diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java index 72ea1ee32..f38cf80b4 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java @@ -1,16 +1,15 @@ package com.tencent.supersonic.auth.api.authentication.adaptor; +import javax.servlet.http.HttpServletRequest; + import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; import java.util.List; import java.util.Set; -import javax.servlet.http.HttpServletRequest; -/** - * UserAdaptor defines some interfaces for obtaining user and organization information - */ +/** UserAdaptor defines some interfaces for obtaining user and organization information */ public interface UserAdaptor { List getUserNames(); diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/annotation/AuthenticationIgnore.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/annotation/AuthenticationIgnore.java index afdb74b12..6752247eb 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/annotation/AuthenticationIgnore.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/annotation/AuthenticationIgnore.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.auth.api.authentication.annotation; - import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -8,6 +7,4 @@ import java.lang.annotation.Target; @Target({ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) -public @interface AuthenticationIgnore { - -} +public @interface AuthenticationIgnore {} diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java index 3983b0074..207532489 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.auth.api.authentication.config; - import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; @@ -25,8 +24,9 @@ public class AuthenticationConfig { @Value("${s2.authentication.token.default.appKey:supersonic}") private String tokenDefaultAppKey; - @Value("${s2.authentication.token.appSecret:supersonic:WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk" - + "783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==}") + @Value( + "${s2.authentication.token.appSecret:supersonic:WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk" + + "783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==}") private String tokenAppSecret; @Value("${s2.authentication.token.http.header.key:Authorization}") diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/constant/UserConstants.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/constant/UserConstants.java index 49737bfa4..590c8ad38 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/constant/UserConstants.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/constant/UserConstants.java @@ -17,5 +17,4 @@ public class UserConstants { public static final String TOKEN_PREFIX = "Bearer"; public static final String INTERNAL = "internal"; - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java index fc909c9f1..3fed959e1 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java @@ -23,5 +23,4 @@ public class Organization { private List subOrganizations = Lists.newArrayList(); private boolean isRoot; - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/User.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/User.java index 438efaa83..8dd4baa83 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/User.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/User.java @@ -20,7 +20,8 @@ public class User { private Integer isAdmin; - public static User get(Long id, String name, String displayName, String email, Integer isAdmin) { + public static User get( + Long id, String name, String displayName, String email, Integer isAdmin) { return new User(id, name, displayName, email, isAdmin); } @@ -44,5 +45,4 @@ public class User { public boolean isSuperAdmin() { return isAdmin != null && isAdmin == 1; } - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/UserWithPassword.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/UserWithPassword.java index 36f77eae2..a8b161028 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/UserWithPassword.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/UserWithPassword.java @@ -9,14 +9,24 @@ public class UserWithPassword extends User { private String password; - public UserWithPassword(Long id, String name, String displayName, String email, String password, Integer isAdmin) { + public UserWithPassword( + Long id, + String name, + String displayName, + String email, + String password, + Integer isAdmin) { super(id, name, displayName, email, isAdmin); this.password = password; } - public static UserWithPassword get(Long id, String name, String displayName, - String email, String password, Integer isAdmin) { + public static UserWithPassword get( + Long id, + String name, + String displayName, + String email, + String password, + Integer isAdmin) { return new UserWithPassword(id, name, displayName, email, password, isAdmin); } - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/request/UserReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/request/UserReq.java index 85e39ccef..bc4b96b91 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/request/UserReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/request/UserReq.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.auth.api.authentication.request; - import javax.validation.constraints.NotBlank; + import lombok.Data; @Data @@ -12,6 +12,4 @@ public class UserReq { @NotBlank(message = "password can not be null") private String password; - - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java index 54d2495bd..393e0531d 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java @@ -1,17 +1,19 @@ package com.tencent.supersonic.auth.api.authentication.service; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; import java.util.Set; public interface UserService { - User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse); + User getCurrentUser( + HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse); List getUserNames(); diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserStrategy.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserStrategy.java index f5106da45..eb867be8a 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserStrategy.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserStrategy.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.auth.api.authentication.service; -import com.tencent.supersonic.auth.api.authentication.pojo.User; - import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.tencent.supersonic.auth.api.authentication.pojo.User; + public interface UserStrategy { boolean accept(boolean isEnableAuthentication); @@ -12,5 +12,4 @@ public interface UserStrategy { User findUser(HttpServletRequest request, HttpServletResponse response); User findUser(String token, String appKey); - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java index 3ea0093d9..78f8e4d38 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java @@ -1,12 +1,13 @@ package com.tencent.supersonic.auth.api.authentication.utils; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; import com.tencent.supersonic.common.config.SystemConfig; import com.tencent.supersonic.common.service.SystemConfigService; import com.tencent.supersonic.common.util.ContextUtils; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.springframework.util.CollectionUtils; public final class UserHolder { @@ -36,5 +37,4 @@ public final class UserHolder { } return user; } - } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java index 683729a3c..5063f0273 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.auth.api.authorization.pojo; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class AuthGroup { @@ -10,18 +11,12 @@ public class AuthGroup { private String name; private Integer groupId; private List authRules; - /** - * row permission expression - */ + /** row permission expression */ private List dimensionFilters; - /** - * row permission expression description information - */ + /** row permission expression description information */ private String dimensionFilterDescription; private List authorizedUsers; - /** - * authorization Department Id - */ + /** authorization Department Id */ private List authorizedDepartmentIds; } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java index fca4b9ade..d2b971a85 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java @@ -10,8 +10,7 @@ public class AuthRes { private Long modelId; private String name; - public AuthRes() { - } + public AuthRes() {} public AuthRes(Long modelId, String name) { this.modelId = modelId; diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRule.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRule.java index b5adafa67..df1d44ede 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRule.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRule.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.auth.api.authorization.pojo; +import lombok.Data; + import java.beans.Transient; import java.util.ArrayList; import java.util.List; -import lombok.Data; @Data public class AuthRule { diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/DimensionFilter.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/DimensionFilter.java index afbbe2d33..102dd01b7 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/DimensionFilter.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/DimensionFilter.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.auth.api.authorization.pojo; +import lombok.Data; + import java.util.ArrayList; import java.util.List; -import lombok.Data; @Data public class DimensionFilter { diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/AddUsersToGroupReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/AddUsersToGroupReq.java index 7135fb636..6c0c2298f 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/AddUsersToGroupReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/AddUsersToGroupReq.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.auth.api.authorization.request; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class AddUsersToGroupReq { diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryGroupReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryGroupReq.java index 0940fcb5b..e5ae135b5 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryGroupReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryGroupReq.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.auth.api.authorization.request; - import com.tencent.supersonic.common.pojo.PageBaseReq; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class QueryGroupReq extends PageBaseReq { diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveGroupReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveGroupReq.java index 6ac974b7f..1f82550c5 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveGroupReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveGroupReq.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.auth.api.authorization.request; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class RemoveGroupReq { diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveUsersFromGroupReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveUsersFromGroupReq.java index 7c4d7e9a1..cbde5e572 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveUsersFromGroupReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/RemoveUsersFromGroupReq.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.auth.api.authorization.request; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class RemoveUsersFromGroupReq { diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/service/AuthService.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/service/AuthService.java index 5f9ed3dd2..f296d9c1a 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/service/AuthService.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/service/AuthService.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq; import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp; + import java.util.List; public interface AuthService { diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java index 53afe18be..41d6f60d3 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.auth.authentication.adaptor; +import javax.servlet.http.HttpServletRequest; + import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor; @@ -9,20 +11,17 @@ import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository; -import com.tencent.supersonic.common.util.AESEncryptionUtil; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; +import com.tencent.supersonic.common.util.AESEncryptionUtil; import com.tencent.supersonic.common.util.ContextUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; -import javax.servlet.http.HttpServletRequest; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -/** - * DefaultUserAdaptor provides a default method to obtain user and organization information - */ +/** DefaultUserAdaptor provides a default method to obtain user and organization information */ @Slf4j public class DefaultUserAdaptor implements UserAdaptor { @@ -49,14 +48,16 @@ public class DefaultUserAdaptor implements UserAdaptor { @Override public List getOrganizationTree() { - Organization superSonic = new Organization("1", "0", - "SuperSonic", "SuperSonic", Lists.newArrayList(), true); - Organization hr = new Organization("2", "1", - "Hr", "SuperSonic/Hr", Lists.newArrayList(), false); - Organization sales = new Organization("3", "1", - "Sales", "SuperSonic/Sales", Lists.newArrayList(), false); - Organization marketing = new Organization("4", "1", - "Marketing", "SuperSonic/Marketing", Lists.newArrayList(), false); + Organization superSonic = + new Organization("1", "0", "SuperSonic", "SuperSonic", Lists.newArrayList(), true); + Organization hr = + new Organization("2", "1", "Hr", "SuperSonic/Hr", Lists.newArrayList(), false); + Organization sales = + new Organization( + "3", "1", "Sales", "SuperSonic/Sales", Lists.newArrayList(), false); + Organization marketing = + new Organization( + "4", "1", "Marketing", "SuperSonic/Marketing", Lists.newArrayList(), false); List subOrganization = Lists.newArrayList(hr, sales, marketing); superSonic.setSubOrganizations(subOrganization); return Lists.newArrayList(superSonic); @@ -112,11 +113,19 @@ public class DefaultUserAdaptor implements UserAdaptor { throw new RuntimeException("user not exist,please register"); } try { - String password = AESEncryptionUtil.encrypt(userReq.getPassword(), - AESEncryptionUtil.getBytesFromString(userDO.getSalt())); + String password = + AESEncryptionUtil.encrypt( + userReq.getPassword(), + AESEncryptionUtil.getBytesFromString(userDO.getSalt())); if (userDO.getPassword().equals(password)) { - UserWithPassword user = UserWithPassword.get(userDO.getId(), userDO.getName(), userDO.getDisplayName(), - userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin()); + UserWithPassword user = + UserWithPassword.get( + userDO.getId(), + userDO.getName(), + userDO.getDisplayName(), + userDO.getEmail(), + userDO.getPassword(), + userDO.getIsAdmin()); return user; } else { throw new RuntimeException("password not correct, please try again"); @@ -135,5 +144,4 @@ public class DefaultUserAdaptor implements UserAdaptor { public Set getUserAllOrgId(String userName) { return Sets.newHashSet(); } - -} \ No newline at end of file +} diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java index 62d83e196..0ba60b894 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.auth.authentication.interceptor; +import javax.servlet.http.HttpServletRequest; + import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.constant.UserConstants; import com.tencent.supersonic.auth.authentication.service.UserServiceImpl; @@ -13,7 +15,6 @@ import org.springframework.util.CollectionUtils; import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest; import org.springframework.web.servlet.HandlerInterceptor; -import javax.servlet.http.HttpServletRequest; import java.lang.reflect.Field; import java.util.Arrays; import java.util.List; @@ -21,14 +22,12 @@ import java.util.List; @Slf4j public abstract class AuthenticationInterceptor implements HandlerInterceptor { - protected AuthenticationConfig authenticationConfig; protected UserServiceImpl userServiceImpl; protected UserTokenUtils userTokenUtils; - protected S2ThreadContext s2ThreadContext; protected boolean isExcludedUri(String uri) { @@ -69,7 +68,8 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor { try { if (request instanceof StandardMultipartHttpServletRequest) { RequestFacade servletRequest = - (RequestFacade) ((StandardMultipartHttpServletRequest) request).getRequest(); + (RequestFacade) + ((StandardMultipartHttpServletRequest) request).getRequest(); Class servletRequestClazz = servletRequest.getClass(); Field request1 = servletRequestClazz.getDeclaredField("request"); request1.setAccessible(true); diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java index 069b2dd04..f7e1dd42b 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.auth.authentication.interceptor; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; @@ -14,15 +16,15 @@ import com.tencent.supersonic.common.util.ThreadContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.web.method.HandlerMethod; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; + import java.lang.reflect.Method; @Slf4j public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor { @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) + public boolean preHandle( + HttpServletRequest request, HttpServletResponse response, Object handler) throws AccessException { authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class); userServiceImpl = ContextUtils.getBean(UserServiceImpl.class); @@ -73,11 +75,11 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor } private void setContext(String userName, HttpServletRequest request) { - ThreadContext threadContext = ThreadContext.builder() - .token(request.getHeader(authenticationConfig.getTokenHttpHeaderKey())) - .userName(userName) - .build(); + ThreadContext threadContext = + ThreadContext.builder() + .token(request.getHeader(authenticationConfig.getTokenHttpHeaderKey())) + .userName(userName) + .build(); s2ThreadContext.set(threadContext); } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java index b754c3980..b688c2fbb 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java @@ -10,20 +10,21 @@ import java.util.List; @Configuration public class InterceptorFactory implements WebMvcConfigurer { - private List authenticationInterceptors; public InterceptorFactory() { - authenticationInterceptors = SpringFactoriesLoader.loadFactories(AuthenticationInterceptor.class, - Thread.currentThread().getContextClassLoader()); + authenticationInterceptors = + SpringFactoriesLoader.loadFactories( + AuthenticationInterceptor.class, + Thread.currentThread().getContextClassLoader()); } @Override public void addInterceptors(InterceptorRegistry registry) { for (AuthenticationInterceptor authenticationInterceptor : authenticationInterceptors) { - registry.addInterceptor(authenticationInterceptor).addPathPatterns("/**") + registry.addInterceptor(authenticationInterceptor) + .addPathPatterns("/**") .excludePathPatterns("/", "/webapp/**", "/error"); } } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java index 759a0b373..4f5fde5a6 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java @@ -1,82 +1,52 @@ package com.tencent.supersonic.auth.authentication.persistence.dataobject; public class UserDO { - /** - * - */ + /** */ private Long id; - /** - * - */ + /** */ private String name; - /** - * - */ + /** */ private String password; private String salt; - /** - * - */ + /** */ private String displayName; - /** - * - */ + /** */ private String email; - /** - * - */ + /** */ private Integer isAdmin; - /** - * - * @return id - */ + /** @return id */ public Long getId() { return id; } - /** - * - * @param id - */ + /** @param id */ public void setId(Long id) { this.id = id; } - /** - * - * @return name - */ + /** @return name */ public String getName() { return name; } - /** - * - * @param name - */ + /** @param name */ public void setName(String name) { this.name = name == null ? null : name.trim(); } - /** - * - * @return password - */ + /** @return password */ public String getPassword() { return password; } - /** - * - * @param password - */ + /** @param password */ public void setPassword(String password) { this.password = password == null ? null : password.trim(); } @@ -89,51 +59,33 @@ public class UserDO { this.salt = salt == null ? null : salt.trim(); } - /** - * - * @return display_name - */ + /** @return display_name */ public String getDisplayName() { return displayName; } - /** - * - * @param displayName - */ + /** @param displayName */ public void setDisplayName(String displayName) { this.displayName = displayName == null ? null : displayName.trim(); } - /** - * - * @return email - */ + /** @return email */ public String getEmail() { return email; } - /** - * - * @param email - */ + /** @param email */ public void setEmail(String email) { this.email = email == null ? null : email.trim(); } - /** - * - * @return is_admin - */ + /** @return is_admin */ public Integer getIsAdmin() { return isAdmin; } - /** - * - * @param isAdmin - */ + /** @param isAdmin */ public void setIsAdmin(Integer isAdmin) { this.isAdmin = isAdmin; } -} \ No newline at end of file +} diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java index 96d8fafdd..7c288784e 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java @@ -4,101 +4,64 @@ import java.util.ArrayList; import java.util.List; public class UserDOExample { - /** - * s2_user - */ + /** s2_user */ protected String orderByClause; - /** - * s2_user - */ + /** s2_user */ protected boolean distinct; - /** - * s2_user - */ + /** s2_user */ protected List oredCriteria; - /** - * s2_user - */ + /** s2_user */ protected Integer limitStart; - /** - * s2_user - */ + /** s2_user */ protected Integer limitEnd; - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public UserDOExample() { oredCriteria = new ArrayList(); } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setOrderByClause(String orderByClause) { this.orderByClause = orderByClause; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public String getOrderByClause() { return orderByClause; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setDistinct(boolean distinct) { this.distinct = distinct; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public boolean isDistinct() { return distinct; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public List getOredCriteria() { return oredCriteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void or(Criteria criteria) { oredCriteria.add(criteria); } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Criteria or() { Criteria criteria = createCriteriaInternal(); oredCriteria.add(criteria); return criteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Criteria createCriteria() { Criteria criteria = createCriteriaInternal(); if (oredCriteria.size() == 0) { @@ -107,60 +70,40 @@ public class UserDOExample { return criteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ protected Criteria createCriteriaInternal() { Criteria criteria = new Criteria(); return criteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void clear() { oredCriteria.clear(); orderByClause = null; distinct = false; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setLimitStart(Integer limitStart) { - this.limitStart=limitStart; + this.limitStart = limitStart; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Integer getLimitStart() { return limitStart; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setLimitEnd(Integer limitEnd) { - this.limitEnd=limitEnd; + this.limitEnd = limitEnd; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Integer getLimitEnd() { return limitEnd; } - /** - * s2_user null - */ + /** s2_user null */ protected abstract static class GeneratedCriteria { protected List criteria; @@ -195,7 +138,8 @@ public class UserDOExample { criteria.add(new Criterion(condition, value)); } - protected void addCriterion(String condition, Object value1, Object value2, String property) { + protected void addCriterion( + String condition, Object value1, Object value2, String property) { if (value1 == null || value2 == null) { throw new RuntimeException("Between values for " + property + " cannot be null"); } @@ -603,9 +547,7 @@ public class UserDOExample { } } - /** - * s2_user - */ + /** s2_user */ public static class Criteria extends GeneratedCriteria { protected Criteria() { @@ -613,9 +555,7 @@ public class UserDOExample { } } - /** - * s2_user null - */ + /** s2_user null */ public static class Criterion { private String condition; @@ -688,7 +628,8 @@ public class UserDOExample { this(condition, value, null); } - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { + protected Criterion( + String condition, Object value, Object secondValue, String typeHandler) { super(); this.condition = condition; this.value = value; @@ -701,4 +642,4 @@ public class UserDOExample { this(condition, value, secondValue, null); } } -} \ No newline at end of file +} diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java index bbda545d2..6eb31b31d 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java @@ -1,22 +1,17 @@ package com.tencent.supersonic.auth.authentication.persistence.mapper; - import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample; -import java.util.List; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + @Mapper public interface UserDOMapper { - /** - * @mbg.generated - */ + /** @mbg.generated */ int insert(UserDO record); - /** - * @mbg.generated - */ + /** @mbg.generated */ List selectByExample(UserDOExample example); - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java index 14037edfb..cb8e332af 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.auth.authentication.persistence.repository; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; + import java.util.List; public interface UserRepository { diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java index 60f8ea67c..e4be6fc72 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.auth.authentication.persistence.repository.impl; - import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample; import com.tencent.supersonic.auth.authentication.persistence.mapper.UserDOMapper; @@ -13,10 +12,8 @@ import java.util.Optional; @Component public class UserRepositoryImpl implements UserRepository { - private UserDOMapper userDOMapper; - public UserRepositoryImpl(UserDOMapper userDOMapper) { this.userDOMapper = userDOMapper; } @@ -39,5 +36,4 @@ public class UserRepositoryImpl implements UserRepository { Optional userDOOptional = userDOS.stream().findFirst(); return userDOOptional.orElse(null); } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java index b56635ba4..d37abddee 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.auth.authentication.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -13,8 +15,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; import java.util.Set; @@ -30,7 +30,8 @@ public class UserController { } @GetMapping("/getCurrentUser") - public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { + public User getCurrentUser( + HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { return userService.getCurrentUser(httpServletRequest, httpServletResponse); } @@ -68,5 +69,4 @@ public class UserController { public String login(@RequestBody UserReq userCmd, HttpServletRequest request) { return userService.login(userCmd, request); } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java index 98f642833..5f321d1c8 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.auth.authentication.service; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; @@ -10,8 +13,7 @@ import com.tencent.supersonic.common.config.SystemConfig; import com.tencent.supersonic.common.service.SystemConfigService; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; + import java.util.List; import java.util.Set; @@ -25,7 +27,8 @@ public class UserServiceImpl implements UserService { } @Override - public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { + public User getCurrentUser( + HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); if (user != null) { SystemConfig systemConfig = sysParameterService.getSystemConfig(); @@ -76,5 +79,4 @@ public class UserServiceImpl implements UserService { public String login(UserReq userReq, String appKey) { return ComponentFactory.getUserAdaptor().login(userReq, appKey); } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java index 818ec779e..a3453139a 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.auth.authentication.strategy; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.springframework.stereotype.Service; @Service @@ -24,5 +24,4 @@ public class FakeUserStrategy implements UserStrategy { public User findUser(String token, String appKey) { return User.getFakeUser(); } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java index 0996b607f..08d571dbe 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java @@ -1,20 +1,18 @@ package com.tencent.supersonic.auth.authentication.strategy; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.springframework.stereotype.Service; - @Service public class HttpHeaderUserStrategy implements UserStrategy { - private final UserTokenUtils userTokenUtils; - public HttpHeaderUserStrategy(UserTokenUtils userTokenUtils) { this.userTokenUtils = userTokenUtils; } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java index 7732a51b2..d1f9758ba 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java @@ -1,26 +1,25 @@ package com.tencent.supersonic.auth.authentication.strategy; +import javax.annotation.PostConstruct; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import java.util.List; -import javax.annotation.PostConstruct; import lombok.Data; import org.springframework.context.annotation.Configuration; +import java.util.List; @Configuration @Data public class UserStrategyFactory { - private List userStrategyList; - private AuthenticationConfig authenticationConfig; - public UserStrategyFactory(AuthenticationConfig authenticationConfig, List userStrategyList) { + public UserStrategyFactory( + AuthenticationConfig authenticationConfig, List userStrategyList) { this.authenticationConfig = authenticationConfig; this.userStrategyList = userStrategyList; } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java index e6cee4640..8176a4604 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.auth.authentication.utils; import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor; import org.springframework.core.io.support.SpringFactoriesLoader; + import java.util.Objects; public class ComponentFactory { @@ -16,8 +17,8 @@ public class ComponentFactory { } private static T init(Class factoryType) { - return SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader()).get(0); + return SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader()) + .get(0); } - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java index 0e7f3ab8a..740e25936 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.auth.authentication.utils; +import javax.crypto.spec.SecretKeySpec; +import javax.servlet.http.HttpServletRequest; + import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; @@ -11,8 +14,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import javax.crypto.spec.SecretKeySpec; -import javax.servlet.http.HttpServletRequest; import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.HashMap; @@ -46,7 +47,9 @@ public class UserTokenUtils { Map claims = new HashMap<>(5); claims.put(TOKEN_USER_ID, user.getId()); claims.put(TOKEN_USER_NAME, StringUtils.isEmpty(user.getName()) ? "" : user.getName()); - claims.put(TOKEN_USER_PASSWORD, StringUtils.isEmpty(user.getPassword()) ? "" : user.getPassword()); + claims.put( + TOKEN_USER_PASSWORD, + StringUtils.isEmpty(user.getPassword()) ? "" : user.getPassword()); claims.put(TOKEN_USER_DISPLAY_NAME, user.getDisplayName()); claims.put(TOKEN_CREATE_TIME, System.currentTimeMillis()); claims.put(TOKEN_IS_ADMIN, user.getIsAdmin()); @@ -79,8 +82,10 @@ public class UserTokenUtils { String userName = String.valueOf(claims.get(TOKEN_USER_NAME)); String email = String.valueOf(claims.get(TOKEN_USER_EMAIL)); String displayName = String.valueOf(claims.get(TOKEN_USER_DISPLAY_NAME)); - Integer isAdmin = claims.get(TOKEN_IS_ADMIN) == null - ? 0 : Integer.parseInt(claims.get(TOKEN_IS_ADMIN).toString()); + Integer isAdmin = + claims.get(TOKEN_IS_ADMIN) == null + ? 0 + : Integer.parseInt(claims.get(TOKEN_IS_ADMIN).toString()); return User.get(userId, userName, displayName, email, isAdmin); } @@ -97,8 +102,10 @@ public class UserTokenUtils { String email = String.valueOf(claims.get(TOKEN_USER_EMAIL)); String displayName = String.valueOf(claims.get(TOKEN_USER_DISPLAY_NAME)); String password = String.valueOf(claims.get(TOKEN_USER_PASSWORD)); - Integer isAdmin = claims.get(TOKEN_IS_ADMIN) == null - ? 0 : Integer.parseInt(claims.get(TOKEN_IS_ADMIN).toString()); + Integer isAdmin = + claims.get(TOKEN_IS_ADMIN) == null + ? 0 + : Integer.parseInt(claims.get(TOKEN_IS_ADMIN).toString()); return UserWithPassword.get(userId, userName, displayName, email, password, isAdmin); } @@ -117,9 +124,12 @@ public class UserTokenUtils { Claims claims; try { String tokenSecret = getTokenSecret(appKey); - claims = Jwts.parser() - .setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8)) - .build().parseClaimsJws(getTokenString(token)).getBody(); + claims = + Jwts.parser() + .setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8)) + .build() + .parseClaimsJws(getTokenString(token)) + .getBody(); } catch (Exception e) { log.error("getClaims", e); throw new AccessException("parse user info from token failed :" + token); @@ -128,8 +138,9 @@ public class UserTokenUtils { } private static String getTokenString(String token) { - return token.startsWith(TOKEN_PREFIX) ? token.substring(token.indexOf(TOKEN_PREFIX) - + TOKEN_PREFIX.length()).trim() : token.trim(); + return token.startsWith(TOKEN_PREFIX) + ? token.substring(token.indexOf(TOKEN_PREFIX) + TOKEN_PREFIX.length()).trim() + : token.trim(); } private String generate(Map claims, String appKey) { @@ -146,8 +157,11 @@ public class UserTokenUtils { .setClaims(claims) .setSubject(claims.get(TOKEN_USER_NAME).toString()) .setExpiration(expirationDate) - .signWith(new SecretKeySpec(tokenSecret.getBytes(StandardCharsets.UTF_8), - SignatureAlgorithm.HS512.getJcaName()), SignatureAlgorithm.HS512) + .signWith( + new SecretKeySpec( + tokenSecret.getBytes(StandardCharsets.UTF_8), + SignatureAlgorithm.HS512.getJcaName()), + SignatureAlgorithm.HS512) .compact(); } diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/rest/AuthController.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/rest/AuthController.java index cea9cbef8..f4e52ff9a 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/rest/AuthController.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/rest/AuthController.java @@ -1,14 +1,14 @@ package com.tencent.supersonic.auth.authorization.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq; import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp; import com.tencent.supersonic.auth.api.authorization.service.AuthService; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; @@ -17,6 +17,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController @RequestMapping("/api/auth") @Slf4j @@ -29,14 +31,13 @@ public class AuthController { } @GetMapping("/queryGroup") - public List queryAuthGroup(@RequestParam("modelId") String modelId, + public List queryAuthGroup( + @RequestParam("modelId") String modelId, @RequestParam(value = "groupId", required = false) Integer groupId) { return authService.queryAuthGroups(modelId, groupId); } - /** - * 新建权限组 - */ + /** 新建权限组 */ @PostMapping("/createGroup") public void newAuthGroup(@RequestBody AuthGroup group) { group.setGroupId(null); @@ -68,9 +69,10 @@ public class AuthController { * @return */ @PostMapping("/queryAuthorizedRes") - public AuthorizedResourceResp queryAuthorizedResources(@RequestBody QueryAuthResReq req, - HttpServletRequest request, - HttpServletResponse response) { + public AuthorizedResourceResp queryAuthorizedResources( + @RequestBody QueryAuthResReq req, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return authService.queryAuthorizedResources(req, user); } diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java index 6e022c105..d6b899e8e 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java @@ -30,23 +30,27 @@ public class AuthServiceImpl implements AuthService { private UserService userService; - public AuthServiceImpl(JdbcTemplate jdbcTemplate, - UserService userService) { + public AuthServiceImpl(JdbcTemplate jdbcTemplate, UserService userService) { this.jdbcTemplate = jdbcTemplate; this.userService = userService; } private List load() { - List rows = jdbcTemplate.queryForList("select config from s2_auth_groups", String.class); + List rows = + jdbcTemplate.queryForList("select config from s2_auth_groups", String.class); Gson g = new Gson(); - return rows.stream().map(row -> g.fromJson(row, AuthGroup.class)).collect(Collectors.toList()); + return rows.stream() + .map(row -> g.fromJson(row, AuthGroup.class)) + .collect(Collectors.toList()); } @Override public List queryAuthGroups(String modelId, Integer groupId) { return load().stream() - .filter(group -> (Objects.isNull(groupId) || groupId.equals(group.getGroupId())) - && modelId.equals(group.getModelId().toString())) + .filter( + group -> + (Objects.isNull(groupId) || groupId.equals(group.getGroupId())) + && modelId.equals(group.getModelId().toString())) .collect(Collectors.toList()); } @@ -61,10 +65,14 @@ public class AuthServiceImpl implements AuthService { nextGroupId = obj + 1; } group.setGroupId(nextGroupId); - jdbcTemplate.update("insert into s2_auth_groups (group_id, config) values (?, ?);", nextGroupId, + jdbcTemplate.update( + "insert into s2_auth_groups (group_id, config) values (?, ?);", + nextGroupId, g.toJson(group)); } else { - jdbcTemplate.update("update s2_auth_groups set config = ? where group_id = ?;", g.toJson(group), + jdbcTemplate.update( + "update s2_auth_groups set config = ? where group_id = ?;", + g.toJson(group), group.getGroupId()); } } @@ -80,10 +88,11 @@ public class AuthServiceImpl implements AuthService { return new AuthorizedResourceResp(); } Set userOrgIds = userService.getUserAllOrgId(user.getName()); - List groups = getAuthGroups(req.getModelIds(), user.getName(), new ArrayList<>(userOrgIds)); + List groups = + getAuthGroups(req.getModelIds(), user.getName(), new ArrayList<>(userOrgIds)); AuthorizedResourceResp resource = new AuthorizedResourceResp(); - Map> authGroupsByModelId = groups.stream() - .collect(Collectors.groupingBy(AuthGroup::getModelId)); + Map> authGroupsByModelId = + groups.stream().collect(Collectors.groupingBy(AuthGroup::getModelId)); for (Long modelId : req.getModelIds()) { if (authGroupsByModelId.containsKey(modelId)) { List authGroups = authGroupsByModelId.get(modelId); @@ -110,26 +119,31 @@ public class AuthServiceImpl implements AuthService { return resource; } - private List getAuthGroups(List modelIds, String userName, List departmentIds) { - List groups = load().stream() - .filter(group -> { - if (!modelIds.contains(group.getModelId())) { - return false; - } - if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) - && group.getAuthorizedUsers().contains(userName)) { - return true; - } - for (String departmentId : departmentIds) { - if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds()) - && group.getAuthorizedDepartmentIds().contains(departmentId)) { - return true; - } - } - return false; - }).collect(Collectors.toList()); + private List getAuthGroups( + List modelIds, String userName, List departmentIds) { + List groups = + load().stream() + .filter( + group -> { + if (!modelIds.contains(group.getModelId())) { + return false; + } + if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) + && group.getAuthorizedUsers().contains(userName)) { + return true; + } + for (String departmentId : departmentIds) { + if (!CollectionUtils.isEmpty( + group.getAuthorizedDepartmentIds()) + && group.getAuthorizedDepartmentIds() + .contains(departmentId)) { + return true; + } + } + return false; + }) + .collect(Collectors.toList()); log.info("user:{} department:{} authGroups:{}", userName, departmentIds, groups); return groups; } - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryReviewResult.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryReviewResult.java index 57bfcc27e..ecea94ef0 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryReviewResult.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryReviewResult.java @@ -1,11 +1,9 @@ package com.tencent.supersonic.chat.api.pojo.enums; - import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import org.apache.commons.lang3.StringUtils; public enum MemoryReviewResult { - POSITIVE, NEGATIVE; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryStatus.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryStatus.java index b00f08b96..6850a3b1d 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryStatus.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/MemoryStatus.java @@ -1,10 +1,7 @@ package com.tencent.supersonic.chat.api.pojo.enums; - public enum MemoryStatus { - PENDING, ENABLED, DISABLED; - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java index f526d8a59..6bfd1af09 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java @@ -7,18 +7,13 @@ import java.util.List; @Data public class ChatAggConfigReq { - /** - * invisible dimensions/metrics - */ + /** invisible dimensions/metrics */ private ItemVisibility visibility; - /** - * information about dictionary about the model - */ + /** information about dictionary about the model */ private List knowledgeInfos; private KnowledgeAdvancedConfig globalKnowledgeConfig; private ChatDefaultConfigReq chatDefaultConfig; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java index 3ad49c784..1384f48b8 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java @@ -1,35 +1,24 @@ package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.common.pojo.enums.StatusEnum; - import lombok.Data; import lombok.ToString; import java.util.List; -/** - * extended information command about model - */ +/** extended information command about model */ @Data @ToString public class ChatConfigBaseReq { private Long modelId; - - /** - * the recommended questions about the model - */ + /** the recommended questions about the model */ private List recommendedQuestions; - /** - * the llm examples about the model - */ + /** the llm examples about the model */ private String llmExamples; - /** - * available status - */ + /** available status */ private StatusEnum status; - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java index 638b03a64..abfb6e41e 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java @@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.api.pojo.request; import lombok.Data; - @Data public class ChatConfigEditReqReq extends ChatConfigBaseReq { private Long id; -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java index cc9095bdf..6b20d1e9a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import lombok.Data; import lombok.NoArgsConstructor; - @NoArgsConstructor @Data public class ChatConfigFilter { @@ -12,4 +11,4 @@ public class ChatConfigFilter { private Long id; private Long modelId; private StatusEnum status = StatusEnum.ONLINE; -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java index a5d9925bd..a2a3fac08 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.api.pojo.request; - import lombok.Data; import java.util.ArrayList; @@ -11,6 +10,4 @@ public class ChatDefaultConfigReq { private List dimensionIds = new ArrayList<>(); private List metricIds = new ArrayList<>(); - - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java index 90356035c..44000bae1 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java @@ -7,18 +7,13 @@ import java.util.List; @Data public class ChatDetailConfigReq { - /** - * invisible dimensions/metrics - */ + /** invisible dimensions/metrics */ private ItemVisibility visibility; - /** - * information about dictionary about the model - */ + /** information about dictionary about the model */ private List knowledgeInfos; private KnowledgeAdvancedConfig globalKnowledgeConfig; private ChatDefaultConfigReq chatDefaultConfig; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java index 58884567d..b3cdff4e8 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java @@ -18,5 +18,4 @@ public class ChatExecuteReq { private int parseId; private String queryText; private boolean saveAnswer; - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryFilter.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryFilter.java index 4bc057416..1ce696866 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryFilter.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryFilter.java @@ -26,5 +26,4 @@ public class ChatMemoryFilter { private MemoryReviewResult llmReviewRet; private MemoryReviewResult humanReviewRet; - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java index 79399f2da..cc76176aa 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java @@ -1,13 +1,12 @@ package com.tencent.supersonic.chat.api.pojo.request; +import javax.validation.constraints.NotNull; import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; -import javax.validation.constraints.NotNull; - @Data public class ChatMemoryUpdateReq extends RecordInfo { @@ -23,5 +22,4 @@ public class ChatMemoryUpdateReq extends RecordInfo { private MemoryReviewResult humanReviewRet; private String humanReviewCmt; - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java index 8ddc57c88..3b98400dc 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java @@ -15,5 +15,4 @@ public class ChatParseReq { private QueryFilters queryFilters; private boolean saveAnswer = true; private SchemaMapInfo mapInfo = new SchemaMapInfo(); - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java index 6848e1dd2..30005d8fd 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java @@ -1,29 +1,22 @@ package com.tencent.supersonic.chat.api.pojo.request; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; -/** - * the entity info about the model - */ +import java.util.List; + +/** the entity info about the model */ @Data @AllArgsConstructor @ToString @NoArgsConstructor public class Entity { - /** - * uniquely identifies an entity - */ + /** uniquely identifies an entity */ private Long entityId; - /** - * entity name list - */ + /** entity name list */ private List names; - - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java index 5ca96f41c..0a5adeddb 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java @@ -6,18 +6,13 @@ import lombok.ToString; import java.util.ArrayList; import java.util.List; - @Data @ToString public class ItemNameVisibilityInfo { - /** - * invisible dimensions - */ + /** invisible dimensions */ private List blackDimNameList = new ArrayList<>(); - /** - * invisible metrics - */ + /** invisible metrics */ private List blackMetricNameList = new ArrayList<>(); } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java index ee3a1eb13..8fac606d4 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java @@ -1,22 +1,18 @@ package com.tencent.supersonic.chat.api.pojo.request; -import java.util.ArrayList; -import java.util.List; import lombok.Data; import lombok.ToString; +import java.util.ArrayList; +import java.util.List; @Data @ToString public class ItemVisibility { - /** - * invisible dimensions - */ + /** invisible dimensions */ private List blackDimIdList = new ArrayList<>(); - /** - * invisible metrics - */ + /** invisible metrics */ private List blackMetricIdList = new ArrayList<>(); -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java index 3c5d8d501..d19033e21 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java @@ -5,13 +5,11 @@ import lombok.Data; import java.util.ArrayList; import java.util.List; -/** - * advanced knowledge config - */ +/** advanced knowledge config */ @Data public class KnowledgeAdvancedConfig { private List blackList = new ArrayList<>(); private List whiteList = new ArrayList<>(); private List ruleList = new ArrayList<>(); -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java index b32e55b81..739e2a042 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java @@ -1,35 +1,23 @@ package com.tencent.supersonic.chat.api.pojo.request; -import com.tencent.supersonic.common.pojo.enums.TypeEnums; - import javax.validation.constraints.NotNull; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import lombok.Data; -/** - * information about dictionary about the model - */ - +/** information about dictionary about the model */ @Data public class KnowledgeInfoReq { - /** - * metricId、DimensionId、modelId - */ + /** metricId、DimensionId、modelId */ private Long itemId; private String bizName; - /** - * type: IntentionTypeEnum - * temporarily only supports dimension-related information - */ - @NotNull - private TypeEnums type = TypeEnums.DIMENSION; + /** type: IntentionTypeEnum temporarily only supports dimension-related information */ + @NotNull private TypeEnums type = TypeEnums.DIMENSION; private Boolean searchEnable = false; - /** - * advanced knowledge config for single item - */ + /** advanced knowledge config for single item */ private KnowledgeAdvancedConfig knowledgeAdvancedConfig; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageMemoryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageMemoryReq.java index fde9aa2d7..19d097630 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageMemoryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageMemoryReq.java @@ -3,10 +3,8 @@ package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.common.pojo.PageBaseReq; import lombok.Data; - @Data public class PageMemoryReq extends PageBaseReq { private ChatMemoryFilter chatMemoryFilter = new ChatMemoryFilter(); - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageQueryInfoReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageQueryInfoReq.java index 952973212..a0e8a3930 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageQueryInfoReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageQueryInfoReq.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.api.pojo.request; import lombok.Data; + import java.util.List; @Data diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java index f06c51531..b14226d9a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java @@ -1,12 +1,10 @@ package com.tencent.supersonic.chat.api.pojo.request; - import lombok.Data; @Data public class PluginQueryReq { - private String name; private String parseMode; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendReq.java index cf2b3585a..a5cf8c2aa 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.api.pojo.request; - import lombok.Data; @Data @@ -9,5 +8,4 @@ public class RecommendReq { private Long modelId; private Long metricId; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java index cbfebb407..9f552b002 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java @@ -12,5 +12,4 @@ import lombok.ToString; public class RecommendedQuestionReq { private String question; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java index 7bcaec731..718c2ed6c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.api.pojo.request; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -21,5 +20,4 @@ public class SimilarQueryReq { private Long dataSetId; private Integer agentId; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java index 33db6e0ec..019d9e443 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java @@ -9,18 +9,13 @@ import java.util.List; @Data public class ChatAggRichConfigResp { - /** - * invisible dimensions/metrics - */ + /** invisible dimensions/metrics */ private ItemVisibilityInfo visibility; - /** - * information about dictionary about the model - */ + /** information about dictionary about the model */ private List knowledgeInfos; private KnowledgeAdvancedConfig globalKnowledgeConfig; private ChatDefaultRichConfigResp chatDefaultConfig; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java index 9b5734794..3f49500ee 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java @@ -4,12 +4,11 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import lombok.Data; import java.util.Date; import java.util.List; -import lombok.Data; - @Data public class ChatConfigResp { @@ -25,13 +24,11 @@ public class ChatConfigResp { private String llmExamples; - /** - * available status - */ + /** available status */ private StatusEnum statusEnum; private String createdBy; private String updatedBy; private Date createdAt; private Date updatedAt; -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java index 89ea1a42e..c9752c1f7 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java @@ -2,11 +2,11 @@ package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import lombok.Data; + import java.util.Date; import java.util.List; -import lombok.Data; - @Data public class ChatConfigRichResp { @@ -23,9 +23,7 @@ public class ChatConfigRichResp { private List recommendedQuestions; - /** - * available status - */ + /** available status */ private StatusEnum statusEnum; private String createdBy; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java index 53bd9222e..1381eb497 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java @@ -1,9 +1,8 @@ package com.tencent.supersonic.chat.api.pojo.response; - -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.Data; import java.util.List; @@ -14,18 +13,11 @@ public class ChatDefaultRichConfigResp { private List dimensions; private List metrics; - - /** - * default time span unit - */ + /** default time span unit */ private Integer unit = 1; - /** - * default time type: - * DAY, WEEK, MONTH, YEAR - */ + /** default time type: DAY, WEEK, MONTH, YEAR */ private String period = Constants.DAY; private TimeMode timeMode; - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java index a56e45106..795e8d3c8 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java @@ -9,19 +9,13 @@ import java.util.List; @Data public class ChatDetailRichConfigResp { - /** - * invisible dimensions/metrics - */ + /** invisible dimensions/metrics */ private ItemVisibilityInfo visibility; - /** - * information about dictionary about the model - */ + /** information about dictionary about the model */ private List knowledgeInfos; private KnowledgeAdvancedConfig globalKnowledgeConfig; private ChatDefaultRichConfigResp chatDefaultConfig; - - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java index d2a2df6ab..8e04dcf5c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java @@ -27,4 +27,4 @@ public class DictLatestTaskResp { private Date createdAt; private Long elapsedMs; -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java index b4731532e..3ed13d59b 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java @@ -1,15 +1,13 @@ package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import lombok.Data; import java.util.List; -import lombok.Data; @Data public class EntityRichInfoResp { - /** - * entity alias - */ + /** entity alias */ private List names; private SchemaElement dimItem; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java index 3bb4e0798..dfd8449e5 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.api.pojo.response; -import java.util.List; import lombok.Data; +import java.util.List; @Data public class ItemVisibilityInfo { @@ -11,4 +11,4 @@ public class ItemVisibilityInfo { private List blackMetricIdList; private List whiteDimIdList; private List whiteMetricIdList; -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java index 2fe2cf67a..1a8f7746b 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java @@ -3,10 +3,10 @@ package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp; import lombok.Data; + import java.util.Date; import java.util.List; - @Data public class QueryResp { @@ -20,5 +20,4 @@ public class QueryResp { private List parseInfos; private List similarQueries; private ParseTimeCostResp parseTimeCost = new ParseTimeCostResp(); - -} \ No newline at end of file +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java index 9b386e3c5..c14f6113b 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.api.pojo.response; - import lombok.Data; import java.util.List; @@ -14,5 +13,4 @@ public class ShowCaseResp { private int pageSize; private int current; - } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java index cde489190..53f67e834 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.api.pojo.response; - import lombok.Builder; import lombok.Data; @@ -11,5 +10,4 @@ public class SimilarQueryRecallResp { private Long queryId; private String queryText; - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java index abb1b11a4..627318011 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.agent; - import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -28,10 +27,9 @@ public class Agent extends RecordInfo { private String name; private String description; - /** - * 0 offline, 1 online - */ + /** 0 offline, 1 online */ private Integer status; + private List examples; private String agentConfig; private ChatModelConfig modelConfig; @@ -46,13 +44,13 @@ public class Agent extends RecordInfo { } List toolList = (List) map.get("tools"); return toolList.stream() - .filter(tool -> { + .filter( + tool -> { if (Objects.isNull(type)) { return true; } return type.name().equals(tool.get("type")); - } - ) + }) .map(JSONObject::toJSONString) .collect(Collectors.toList()); } @@ -74,7 +72,8 @@ public class Agent extends RecordInfo { if (CollectionUtils.isEmpty(tools)) { return Lists.newArrayList(); } - return tools.stream().map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class)) + return tools.stream() + .map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class)) .collect(Collectors.toList()); } @@ -121,7 +120,8 @@ public class Agent extends RecordInfo { if (CollectionUtils.isEmpty(commonAgentTools)) { return new HashSet<>(); } - return commonAgentTools.stream().map(NL2SQLTool::getDataSetIds) + return commonAgentTools.stream() + .map(NL2SQLTool::getDataSetIds) .filter(modelIds -> !CollectionUtils.isEmpty(modelIds)) .flatMap(Collection::stream) .collect(Collectors.toSet()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java index eb460d56d..58d6d5a29 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java @@ -4,8 +4,8 @@ import com.google.common.collect.Lists; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; +import java.util.List; @Data @NoArgsConstructor @@ -13,5 +13,4 @@ import java.util.List; public class AgentConfig { List tools = Lists.newArrayList(); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java index 8a74be690..ba04f4d21 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java @@ -21,5 +21,4 @@ public enum AgentToolType { map.put(PLUGIN, PLUGIN.title); return map; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java index a582f4475..4921d0969 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java @@ -8,5 +8,4 @@ import java.util.List; public class LLMParserTool extends NL2SQLTool { private List exampleQuestions; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/MultiTurnConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/MultiTurnConfig.java index c290cf262..d7c67bc30 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/MultiTurnConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/MultiTurnConfig.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.agent; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -11,5 +10,4 @@ import lombok.NoArgsConstructor; public class MultiTurnConfig { private boolean enableMultiTurn; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java index 23df168d2..59f48bb38 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.agent; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -13,5 +12,4 @@ import java.util.List; public class NL2SQLTool extends AgentTool { protected List dataSetIds; - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java index 485332eff..c83c82ac5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.agent; - import lombok.Data; import java.util.List; @@ -9,5 +8,4 @@ import java.util.List; public class PluginTool extends AgentTool { private List plugins; - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java index 37dc7d4f7..0596f7ad0 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.agent; - import lombok.Data; import org.apache.commons.collections.CollectionUtils; @@ -9,7 +8,6 @@ import java.util.List; @Data public class RuleParserTool extends NL2SQLTool { - private List queryModes; private List queryTypes; @@ -17,5 +15,4 @@ public class RuleParserTool extends NL2SQLTool { public boolean isContainsAllModel() { return CollectionUtils.isNotEmpty(dataSetIds) && dataSetIds.contains(-1L); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfig.java index 205176e72..77de90142 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfig.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.chat.server.config; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.RecordInfo; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import lombok.Data; import lombok.ToString; @@ -14,33 +14,22 @@ import java.util.List; @ToString public class ChatConfig { - /** - * database auto-increment primary key - */ + /** database auto-increment primary key */ private Long id; private Long modelId; - /** - * the chatDetailConfig about the model - */ + /** the chatDetailConfig about the model */ private ChatDetailConfigReq chatDetailConfig; - /** - * the chatAggConfig about the model - */ + /** the chatAggConfig about the model */ private ChatAggConfigReq chatAggConfig; private List recommendedQuestions; - /** - * available status - */ + /** available status */ private StatusEnum status; - /** - * about createdBy, createdAt, updatedBy, updatedAt - */ + /** about createdBy, createdAt, updatedBy, updatedAt */ private RecordInfo recordInfo; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfigFilterInternal.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfigFilterInternal.java index e4f9a614c..8dc17a3c9 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfigFilterInternal.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/config/ChatConfigFilterInternal.java @@ -8,4 +8,4 @@ public class ChatConfigFilterInternal { private Long id; private Long modelId; private Integer status; -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/ChatQueryExecutor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/ChatQueryExecutor.java index e478d9c35..0a2115820 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/ChatQueryExecutor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/ChatQueryExecutor.java @@ -1,10 +1,9 @@ package com.tencent.supersonic.chat.server.executor; -import com.tencent.supersonic.chat.server.pojo.ExecuteContext; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.server.pojo.ExecuteContext; public interface ChatQueryExecutor { QueryResult execute(ExecuteContext executeContext); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PlainTextExecutor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PlainTextExecutor.java index 79eb00a10..7427f0878 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PlainTextExecutor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PlainTextExecutor.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.server.executor; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.agent.MultiTurnConfig; import com.tencent.supersonic.chat.server.parser.ParserConfig; @@ -8,7 +9,6 @@ import com.tencent.supersonic.chat.server.pojo.ExecuteContext; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatManageService; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.ChatLanguageModel; @@ -26,13 +26,14 @@ import static com.tencent.supersonic.chat.server.parser.ParserConfig.PARSER_MULT public class PlainTextExecutor implements ChatQueryExecutor { - private static final String INSTRUCTION = "" - + "#Role: You are a nice person to talk to.\n" - + "#Task: Respond quickly and nicely to the user." - + "#Rules: 1.ALWAYS use the same language as the input.\n" - + "#History Inputs: %s\n" - + "#Current Input: %s\n" - + "#Your response: "; + private static final String INSTRUCTION = + "" + + "#Role: You are a nice person to talk to.\n" + + "#Task: Respond quickly and nicely to the user." + + "#Rules: 1.ALWAYS use the same language as the input.\n" + + "#History Inputs: %s\n" + + "#Current Input: %s\n" + + "#Your response: "; @Override public QueryResult execute(ExecuteContext executeContext) { @@ -40,14 +41,18 @@ public class PlainTextExecutor implements ChatQueryExecutor { return null; } - String promptStr = String.format(INSTRUCTION, getHistoryInputs(executeContext), - executeContext.getQueryText()); + String promptStr = + String.format( + INSTRUCTION, + getHistoryInputs(executeContext), + executeContext.getQueryText()); Prompt prompt = PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP); AgentService agentService = ContextUtils.getBean(AgentService.class); Agent chatAgent = agentService.getAgent(executeContext.getAgent().getId()); - ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatAgent.getModelConfig()); + ChatLanguageModel chatLanguageModel = + ModelProvider.getChatModel(chatAgent.getModelConfig()); Response response = chatLanguageModel.generate(prompt.toUserMessage()); QueryResult result = new QueryResult(); @@ -66,16 +71,21 @@ public class PlainTextExecutor implements ChatQueryExecutor { ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class); MultiTurnConfig agentMultiTurnConfig = chatAgent.getMultiTurnConfig(); - Boolean globalMultiTurnConfig = Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE)); - Boolean multiTurnConfig = agentMultiTurnConfig != null - ? agentMultiTurnConfig.isEnableMultiTurn() : globalMultiTurnConfig; + Boolean globalMultiTurnConfig = + Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE)); + Boolean multiTurnConfig = + agentMultiTurnConfig != null + ? agentMultiTurnConfig.isEnableMultiTurn() + : globalMultiTurnConfig; if (Boolean.TRUE.equals(multiTurnConfig)) { List queryResps = getHistoryQueries(executeContext.getChatId(), 5); - queryResps.stream().forEach(p -> { - historyInput.append(p.getQueryText()); - historyInput.append(";"); - }); + queryResps.stream() + .forEach( + p -> { + historyInput.append(p.getQueryText()); + historyInput.append(";"); + }); } return historyInput.toString(); @@ -83,17 +93,20 @@ public class PlainTextExecutor implements ChatQueryExecutor { private List getHistoryQueries(int chatId, int multiNum) { ChatManageService chatManageService = ContextUtils.getBean(ChatManageService.class); - List contextualParseInfoList = chatManageService.getChatQueries(chatId) - .stream() - .filter(q -> Objects.nonNull(q.getQueryResult()) - && q.getQueryResult().getQueryState() == QueryState.SUCCESS) - .collect(Collectors.toList()); + List contextualParseInfoList = + chatManageService.getChatQueries(chatId).stream() + .filter( + q -> + Objects.nonNull(q.getQueryResult()) + && q.getQueryResult().getQueryState() + == QueryState.SUCCESS) + .collect(Collectors.toList()); - List contextualList = contextualParseInfoList.subList(0, - Math.min(multiNum, contextualParseInfoList.size())); + List contextualList = + contextualParseInfoList.subList( + 0, Math.min(multiNum, contextualParseInfoList.size())); Collections.reverse(contextualList); return contextualList; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PluginExecutor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PluginExecutor.java index 745c60cf1..90278f54e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PluginExecutor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PluginExecutor.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat.server.executor; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.plugin.PluginQueryManager; import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery; import com.tencent.supersonic.chat.server.pojo.ExecuteContext; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; public class PluginExecutor implements ChatQueryExecutor { @@ -18,5 +18,4 @@ public class PluginExecutor implements ChatQueryExecutor { query.setParseInfo(parseInfo); return query.build(); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java index 93c63a0ab..2abdea5b7 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java @@ -1,8 +1,11 @@ package com.tencent.supersonic.chat.server.executor; import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; +import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.chat.server.pojo.ExecuteContext; +import com.tencent.supersonic.chat.server.service.ChatContextService; import com.tencent.supersonic.chat.server.service.MemoryService; import com.tencent.supersonic.chat.server.util.ResultFormatter; import com.tencent.supersonic.common.pojo.Text2SQLExemplar; @@ -10,13 +13,10 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; -import com.tencent.supersonic.chat.server.service.ChatContextService; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; @@ -31,28 +31,35 @@ public class SqlExecutor implements ChatQueryExecutor { QueryResult queryResult = doExecute(executeContext); if (queryResult != null) { - String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(), - queryResult.getQueryResults()); + String textResult = + ResultFormatter.transform2TextNew( + queryResult.getQueryColumns(), queryResult.getQueryResults()); queryResult.setTextResult(textResult); if (queryResult.getQueryState().equals(QueryState.SUCCESS) && queryResult.getQueryMode().equals(LLMSqlQuery.QUERY_MODE)) { - Text2SQLExemplar exemplar = JsonUtil.toObject(JsonUtil.toString( - executeContext.getParseInfo().getProperties() - .get(Text2SQLExemplar.PROPERTY_KEY)), Text2SQLExemplar.class); + Text2SQLExemplar exemplar = + JsonUtil.toObject( + JsonUtil.toString( + executeContext + .getParseInfo() + .getProperties() + .get(Text2SQLExemplar.PROPERTY_KEY)), + Text2SQLExemplar.class); MemoryService memoryService = ContextUtils.getBean(MemoryService.class); - memoryService.createMemory(ChatMemoryDO.builder() - .agentId(executeContext.getAgent().getId()) - .status(MemoryStatus.PENDING) - .question(exemplar.getQuestion()) - .sideInfo(exemplar.getSideInfo()) - .dbSchema(exemplar.getDbSchema()) - .s2sql(exemplar.getSql()) - .createdBy(executeContext.getUser().getName()) - .updatedBy(executeContext.getUser().getName()) - .createdAt(new Date()) - .build()); + memoryService.createMemory( + ChatMemoryDO.builder() + .agentId(executeContext.getAgent().getId()) + .status(MemoryStatus.PENDING) + .question(exemplar.getQuestion()) + .sideInfo(exemplar.getSideInfo()) + .dbSchema(exemplar.getDbSchema()) + .s2sql(exemplar.getSql()) + .createdBy(executeContext.getUser().getName()) + .updatedBy(executeContext.getUser().getName()) + .createdAt(new Date()) + .build()); } } @@ -71,9 +78,8 @@ public class SqlExecutor implements ChatQueryExecutor { return null; } - QuerySqlReq sqlReq = QuerySqlReq.builder() - .sql(parseInfo.getSqlInfo().getCorrectedS2SQL()) - .build(); + QuerySqlReq sqlReq = + QuerySqlReq.builder().sql(parseInfo.getSqlInfo().getCorrectedS2SQL()).build(); sqlReq.setSqlInfo(parseInfo.getSqlInfo()); sqlReq.setDataSetId(parseInfo.getDataSetId()); @@ -97,5 +103,4 @@ public class SqlExecutor implements ChatQueryExecutor { } return queryResult; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java index ad3bea715..2946f9af5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java @@ -27,27 +27,26 @@ public class MemoryReviewTask { private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); - private static final String INSTRUCTION = "" - + "#Role: You are a senior data engineer experienced in writing SQL.\n" - + "#Task: Your will be provided with a user question and the SQL written by junior engineer," - + "please take a review and give your opinion.\n" - + "#Rules: " - + "1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),comment=(your comment)`." - + "2.ALWAYS recognize `数据日期` as the date field." - + "3.IGNORE `数据日期` if not expressed in the `Question`." - + "#Question: %s\n" - + "#Schema: %s\n" - + "#SideInfo: %s\n" - + "#SQL: %s\n" - + "#Response: "; + private static final String INSTRUCTION = + "" + + "#Role: You are a senior data engineer experienced in writing SQL.\n" + + "#Task: Your will be provided with a user question and the SQL written by junior engineer," + + "please take a review and give your opinion.\n" + + "#Rules: " + + "1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),comment=(your comment)`." + + "2.ALWAYS recognize `数据日期` as the date field." + + "3.IGNORE `数据日期` if not expressed in the `Question`." + + "#Question: %s\n" + + "#Schema: %s\n" + + "#SideInfo: %s\n" + + "#SQL: %s\n" + + "#Response: "; private static final Pattern OUTPUT_PATTERN = Pattern.compile("opinion=(.*),.*comment=(.*)"); - @Autowired - private MemoryService memoryService; + @Autowired private MemoryService memoryService; - @Autowired - private AgentService agentService; + @Autowired private AgentService agentService; @Scheduled(fixedDelay = 60 * 1000) public void review() { @@ -68,7 +67,8 @@ public class MemoryReviewTask { Prompt prompt = PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP); keyPipelineLog.info("MemoryReviewTask reqPrompt:\n{}", promptStr); - ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatAgent.getModelConfig()); + ChatLanguageModel chatLanguageModel = + ModelProvider.getChatModel(chatAgent.getModelConfig()); if (Objects.nonNull(chatLanguageModel)) { String response = chatLanguageModel.generate(prompt.toUserMessage()).content().text(); keyPipelineLog.info("MemoryReviewTask modelResp:\n{}", response); @@ -79,7 +79,8 @@ public class MemoryReviewTask { } private String createPromptString(ChatMemoryDO m) { - return String.format(INSTRUCTION, m.getQuestion(), m.getDbSchema(), m.getSideInfo(), m.getS2sql()); + return String.format( + INSTRUCTION, m.getQuestion(), m.getDbSchema(), m.getSideInfo(), m.getS2sql()); } private void processResponse(String response, ChatMemoryDO m) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ChatQueryParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ChatQueryParser.java index 6fa9291a5..f420f7229 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ChatQueryParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ChatQueryParser.java @@ -6,5 +6,4 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp; public interface ChatQueryParser { void parse(ParseContext parseContext, ParseResp parseResp); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2PluginParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2PluginParser.java index 456a69e54..1cbfa9311 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2PluginParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2PluginParser.java @@ -6,12 +6,14 @@ import com.tencent.supersonic.chat.server.util.ComponentFactory; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import lombok.extern.slf4j.Slf4j; + import java.util.List; @Slf4j public class NL2PluginParser implements ChatQueryParser { - private final List pluginRecognizers = ComponentFactory.getPluginRecognizers(); + private final List pluginRecognizers = + ComponentFactory.getPluginRecognizers(); @Override public void parse(ParseContext parseContext, ParseResp parseResp) { @@ -19,11 +21,13 @@ public class NL2PluginParser implements ChatQueryParser { return; } - pluginRecognizers.forEach(pluginRecognizer -> { - pluginRecognizer.recognize(parseContext, parseResp); - log.info("{} recallResult:{}", pluginRecognizer.getClass().getSimpleName(), - JsonUtil.toString(parseResp)); - }); + pluginRecognizers.forEach( + pluginRecognizer -> { + pluginRecognizer.recognize(parseContext, parseResp); + log.info( + "{} recallResult:{}", + pluginRecognizer.getClass().getSimpleName(), + JsonUtil.toString(parseResp)); + }); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java index 0f3ad22d2..4e7870a2e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.server.parser; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.server.agent.MultiTurnConfig; import com.tencent.supersonic.chat.server.plugin.PluginQueryManager; +import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.chat.server.pojo.ParseContext; import com.tencent.supersonic.chat.server.service.ChatContextService; import com.tencent.supersonic.chat.server.service.ChatManageService; @@ -19,7 +20,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq; import com.tencent.supersonic.headless.api.pojo.response.MapResp; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; import dev.langchain4j.data.message.AiMessage; @@ -50,31 +50,33 @@ public class NL2SQLParser implements ChatQueryParser { private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); - private static final String REWRITE_USER_QUESTION_INSTRUCTION = "" - + "#Role: You are a data product manager experienced in data requirements." - + "#Task: Your will be provided with current and history questions asked by a user," - + "along with their mapped schema elements(metric, dimension and value)," - + "please try understanding the semantics and rewrite a question." - + "#Rules: " - + "1.ALWAYS keep relevant entities, metrics, dimensions, values and date ranges." - + "2.ONLY respond with the rewritten question." - + "#Current Question: {{current_question}}" - + "#Current Mapped Schema: {{current_schema}}" - + "#History Question: {{history_question}}" - + "#History Mapped Schema: {{history_schema}}" - + "#History SQL: {{history_sql}}" - + "#Rewritten Question: "; + private static final String REWRITE_USER_QUESTION_INSTRUCTION = + "" + + "#Role: You are a data product manager experienced in data requirements." + + "#Task: Your will be provided with current and history questions asked by a user," + + "along with their mapped schema elements(metric, dimension and value)," + + "please try understanding the semantics and rewrite a question." + + "#Rules: " + + "1.ALWAYS keep relevant entities, metrics, dimensions, values and date ranges." + + "2.ONLY respond with the rewritten question." + + "#Current Question: {{current_question}}" + + "#Current Mapped Schema: {{current_schema}}" + + "#History Question: {{history_question}}" + + "#History Mapped Schema: {{history_schema}}" + + "#History SQL: {{history_sql}}" + + "#Rewritten Question: "; - private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION = "" - + "#Role: You are a data business partner who closely interacts with business people.\n" - + "#Task: Your will be provided with user input, system output and some examples, " - + "please respond shortly to teach user how to ask the right question, " - + "by using `Examples` as references." - + "#Rules: ALWAYS respond with the same language as the `Input`.\n" - + "#Input: {{user_question}}\n" - + "#Output: {{system_message}}\n" - + "#Examples: {{examples}}\n" - + "#Response: "; + private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION = + "" + + "#Role: You are a data business partner who closely interacts with business people.\n" + + "#Task: Your will be provided with user input, system output and some examples, " + + "please respond shortly to teach user how to ask the right question, " + + "by using `Examples` as references." + + "#Rules: ALWAYS respond with the same language as the `Input`.\n" + + "#Input: {{user_question}}\n" + + "#Output: {{system_message}}\n" + + "#Examples: {{examples}}\n" + + "#Response: "; @Override public void parse(ParseContext parseContext, ParseResp parseResp) { @@ -84,8 +86,8 @@ public class NL2SQLParser implements ChatQueryParser { ChatContextService chatContextService = ContextUtils.getBean(ChatContextService.class); ChatContext chatCtx = chatContextService.getOrCreateContext(parseContext.getChatId()); - ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel( - parseContext.getAgent().getModelConfig()); + ChatLanguageModel chatLanguageModel = + ModelProvider.getChatModel(parseContext.getAgent().getModelConfig()); processMultiTurn(chatLanguageModel, parseContext); QueryNLReq queryNLReq = QueryReqConverter.buildText2SqlQueryReq(parseContext, chatCtx); @@ -96,11 +98,13 @@ public class NL2SQLParser implements ChatQueryParser { if (ParseResp.ParseState.COMPLETED.equals(text2SqlParseResp.getState())) { parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses()); } else { - parseResp.setErrorMsg(rewriteErrorMessage(chatLanguageModel, - parseContext.getQueryText(), - text2SqlParseResp.getErrorMsg(), - queryNLReq.getDynamicExemplars(), - parseContext.getAgent().getExamples())); + parseResp.setErrorMsg( + rewriteErrorMessage( + chatLanguageModel, + parseContext.getQueryText(), + text2SqlParseResp.getErrorMsg(), + queryNLReq.getDynamicExemplars(), + parseContext.getAgent().getExamples())); } parseResp.setState(text2SqlParseResp.getState()); parseResp.getParseTimeCost().setSqlTime(text2SqlParseResp.getParseTimeCost().getSqlTime()); @@ -134,24 +138,35 @@ public class NL2SQLParser implements ChatQueryParser { StringBuilder textBuilder = new StringBuilder(); textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" "); Optional metric = parseInfo.getMetrics().stream().findFirst(); - metric.ifPresent(schemaElement -> - textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" ")); - List dimensionNames = parseInfo.getDimensions().stream() - .map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList()); + metric.ifPresent( + schemaElement -> + textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" ")); + List dimensionNames = + parseInfo.getDimensions().stream() + .map(SchemaElement::getName) + .filter(Objects::nonNull) + .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(dimensionNames)) { textBuilder.append("**维度:** ").append(String.join(",", dimensionNames)); } textBuilder.append("\n\n**筛选条件:** \n"); if (parseInfo.getDateInfo() != null) { - textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate()).append("~") - .append(parseInfo.getDateInfo().getEndDate()).append(" "); + textBuilder + .append("**数据时间:** ") + .append(parseInfo.getDateInfo().getStartDate()) + .append("~") + .append(parseInfo.getDateInfo().getEndDate()) + .append(" "); } if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters()) || CollectionUtils.isEmpty(parseInfo.getMetricFilters())) { Set queryFilters = parseInfo.getDimensionFilters(); queryFilters.addAll(parseInfo.getMetricFilters()); for (QueryFilter queryFilter : queryFilters) { - textBuilder.append("**").append(queryFilter.getName()).append("**") + textBuilder + .append("**") + .append(queryFilter.getName()) + .append("**") .append(" ") .append(queryFilter.getOperator().getValue()) .append(" ") @@ -165,10 +180,13 @@ public class NL2SQLParser implements ChatQueryParser { private void processMultiTurn(ChatLanguageModel chatLanguageModel, ParseContext parseContext) { ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class); MultiTurnConfig agentMultiTurnConfig = parseContext.getAgent().getMultiTurnConfig(); - Boolean globalMultiTurnConfig = Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE)); + Boolean globalMultiTurnConfig = + Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE)); - Boolean multiTurnConfig = agentMultiTurnConfig != null - ? agentMultiTurnConfig.isEnableMultiTurn() : globalMultiTurnConfig; + Boolean multiTurnConfig = + agentMultiTurnConfig != null + ? agentMultiTurnConfig.isEnableMultiTurn() + : globalMultiTurnConfig; if (!Boolean.TRUE.equals(multiTurnConfig)) { return; } @@ -186,7 +204,8 @@ public class NL2SQLParser implements ChatQueryParser { SemanticParseInfo lastParseInfo = lastQuery.getParseInfos().get(0); Long dataId = lastParseInfo.getDataSetId(); - String curtMapStr = generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId)); + String curtMapStr = + generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId)); String histMapStr = generateSchemaPrompt(lastParseInfo.getElementMatches()); String histSQL = lastParseInfo.getSqlInfo().getCorrectedS2SQL(); @@ -207,22 +226,31 @@ public class NL2SQLParser implements ChatQueryParser { QueryNLReq rewrittenQueryNLReq = QueryReqConverter.buildText2SqlQueryReq(parseContext); MapResp rewrittenQueryMapResult = chatLayerService.performMapping(rewrittenQueryNLReq); parseContext.setMapInfo(rewrittenQueryMapResult.getMapInfo()); - log.info("Last Query: {} Current Query: {}, Rewritten Query: {}", - lastQuery.getQueryText(), currentMapResult.getQueryText(), rewrittenQuery); + log.info( + "Last Query: {} Current Query: {}, Rewritten Query: {}", + lastQuery.getQueryText(), + currentMapResult.getQueryText(), + rewrittenQuery); } - private String rewriteErrorMessage(ChatLanguageModel chatLanguageModel, String userQuestion, - String errMsg, List similarExemplars, - List agentExamples) { + private String rewriteErrorMessage( + ChatLanguageModel chatLanguageModel, + String userQuestion, + String errMsg, + List similarExemplars, + List agentExamples) { Map variables = new HashMap<>(); variables.put("user_question", userQuestion); variables.put("system_message", errMsg); StringBuilder exampleStr = new StringBuilder(); - similarExemplars.forEach(e -> - exampleStr.append(String.format(" ", e.getQuestion(), e.getDbSchema()))); - agentExamples.forEach(e -> - exampleStr.append(String.format(" ", e))); + similarExemplars.forEach( + e -> + exampleStr.append( + String.format( + " ", + e.getQuestion(), e.getDbSchema()))); + agentExamples.forEach(e -> exampleStr.append(String.format(" ", e))); variables.put("examples", exampleStr); Prompt prompt = PromptTemplate.from(REWRITE_ERROR_MESSAGE_INSTRUCTION).apply(variables); @@ -262,14 +290,18 @@ public class NL2SQLParser implements ChatQueryParser { private List getHistoryQueries(int chatId, int multiNum) { ChatManageService chatManageService = ContextUtils.getBean(ChatManageService.class); - List contextualParseInfoList = chatManageService.getChatQueries(chatId) - .stream() - .filter(q -> Objects.nonNull(q.getQueryResult()) - && q.getQueryResult().getQueryState() == QueryState.SUCCESS) - .collect(Collectors.toList()); + List contextualParseInfoList = + chatManageService.getChatQueries(chatId).stream() + .filter( + q -> + Objects.nonNull(q.getQueryResult()) + && q.getQueryResult().getQueryState() + == QueryState.SUCCESS) + .collect(Collectors.toList()); - List contextualList = contextualParseInfoList.subList(0, - Math.min(multiNum, contextualParseInfoList.size())); + List contextualList = + contextualParseInfoList.subList( + 0, Math.min(multiNum, contextualParseInfoList.size())); Collections.reverse(contextualList); return contextualList; } @@ -278,9 +310,8 @@ public class NL2SQLParser implements ChatQueryParser { ExemplarServiceImpl exemplarManager = ContextUtils.getBean(ExemplarServiceImpl.class); EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); String memoryCollectionName = embeddingConfig.getMemoryCollectionName(agentId); - List exemplars = exemplarManager.recallExemplars(memoryCollectionName, - queryNLReq.getQueryText(), 5); + List exemplars = + exemplarManager.recallExemplars(memoryCollectionName, queryNLReq.getQueryText(), 5); queryNLReq.getDynamicExemplars().addAll(exemplars); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ParserConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ParserConfig.java index 9f96f74fc..b19dc074f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ParserConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/ParserConfig.java @@ -10,8 +10,11 @@ import org.springframework.stereotype.Service; public class ParserConfig extends ParameterConfig { public static final Parameter PARSER_MULTI_TURN_ENABLE = - new Parameter("s2.parser.multi-turn.enable", "false", - "是否开启多轮对话", "开启多轮对话将消耗更多token", - "bool", "Parser相关配置"); - + new Parameter( + "s2.parser.multi-turn.enable", + "false", + "是否开启多轮对话", + "开启多轮对话将消耗更多token", + "bool", + "Parser相关配置"); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/PlainTextParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/PlainTextParser.java index 7f5de681b..a73f66ebc 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/PlainTextParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/PlainTextParser.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.chat.server.pojo.ParseContext; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; - public class PlainTextParser implements ChatQueryParser { @Override @@ -18,5 +17,4 @@ public class PlainTextParser implements ChatQueryParser { parseResp.getSelectedParses().add(parseInfo); parseResp.setState(ParseResp.ParseState.COMPLETED); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java index b57d55952..e9b03ead1 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java @@ -10,61 +10,40 @@ import java.util.Date; @Data @TableName("s2_agent") public class AgentDO { - /** - * - */ + /** */ @TableId(type = IdType.AUTO) private Integer id; - /** - * - */ + /** */ private String name; - /** - * - */ + /** */ private String description; - /** - * 0 offline, 1 online - */ + /** 0 offline, 1 online */ private Integer status; - /** - * - */ + /** */ private String examples; - /** - * - */ + /** */ private String config; - /** - * - */ + /** */ private String createdBy; - /** - * - */ + /** */ private Date createdAt; - /** - * - */ + /** */ private String updatedBy; - /** - * - */ + /** */ private Date updatedAt; - /** - * - */ + /** */ private Integer enableSearch; + private Integer enableMemoryReview; private String modelConfig; private String multiTurnConfig; @@ -72,5 +51,4 @@ public class AgentDO { private String visualConfig; private String promptConfig; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatConfigDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatConfigDO.java index 9f9146063..dfc166bd7 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatConfigDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatConfigDO.java @@ -1,18 +1,15 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; -import java.util.Date; - import lombok.Data; import lombok.ToString; +import java.util.Date; @Data @ToString public class ChatConfigDO { - /** - * database auto-increment primary key - */ + /** database auto-increment primary key */ private Long id; private Long modelId; @@ -27,12 +24,10 @@ public class ChatConfigDO { private String llmExamples; - /** - * record info - */ + /** record info */ private String createdBy; + private String updatedBy; private Date createdAt; private Date updatedAt; - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java index 1241a6928..d03ca14b8 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java @@ -61,5 +61,4 @@ public class ChatMemoryDO { @TableField("updated_at") private Date updatedAt; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatQueryDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatQueryDO.java index 03e7017b1..9cecca22b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatQueryDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatQueryDO.java @@ -4,54 +4,44 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; + import java.util.Date; @Data @TableName("s2_chat_query") public class ChatQueryDO { - /** - */ + /** */ @TableId(type = IdType.AUTO) private Long questionId; - /** - */ + /** */ private Integer agentId; - /** - */ + /** */ private Date createTime; - /** - */ + /** */ private String userName; - /** - */ + /** */ private Integer queryState; - /** - */ + /** */ private Long chatId; - /** - */ + /** */ private Integer score; - /** - */ + /** */ private String feedback; - /** - */ + /** */ private String queryText; - /** - */ + /** */ private String queryResult; private String similarQueries; private String parseTimeCost; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictConfDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictConfDO.java index e39a0d3b4..78a65bbec 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictConfDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictConfDO.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; -import java.util.Date; - import lombok.Data; +import java.util.Date; + @Data public class DictConfDO { @@ -17,5 +17,4 @@ public class DictConfDO { private String updatedBy; private Date createdAt; private Date updatedAt; - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictTaskDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictTaskDO.java index be38e717a..93817593d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictTaskDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DictTaskDO.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; -import java.util.Date; - import lombok.Data; import lombok.ToString; import org.apache.commons.codec.digest.DigestUtils; +import java.util.Date; + @Data @ToString public class DictTaskDO { @@ -35,4 +35,4 @@ public class DictTaskDO { public String getCommandMd5() { return DigestUtils.md5Hex(command); } -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java index 21f63dcc6..e0414efe8 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; - import com.tencent.supersonic.headless.core.config.DefaultMetric; import com.tencent.supersonic.headless.core.config.Dim4Dict; import lombok.Data; @@ -9,7 +8,6 @@ import lombok.ToString; import java.util.ArrayList; import java.util.List; - @Data @ToString public class DimValueDO { @@ -34,4 +32,4 @@ public class DimValueDO { this.dimensions = dimensions; return this; } -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java index c4efb4744..bbb834757 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; + import java.util.Date; @Data @@ -36,5 +37,4 @@ public class PluginDO { private String config; private String comment; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java index cd2ee5da4..216d6518a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.Builder; +import lombok.Data; import lombok.Getter; +import lombok.NoArgsConstructor; import java.util.Date; @@ -14,40 +14,25 @@ import java.util.Date; @Getter @AllArgsConstructor public class StatisticsDO { - /** - * questionId - */ + /** questionId */ private Long questionId; - /** - * chatId - */ + /** chatId */ private Long chatId; - /** - * createTime - */ + /** createTime */ private Date createTime; - /** - * queryText - */ + /** queryText */ private String queryText; - /** - * userName - */ + /** userName */ private String userName; - - /** - * interface - */ + /** interface */ private String interfaceName; - /** - * cost - */ + /** cost */ private Integer cost; private Integer type; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/AgentDOMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/AgentDOMapper.java index 4334782e4..4b751135e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/AgentDOMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/AgentDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface AgentDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface AgentDOMapper extends BaseMapper {} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatConfigMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatConfigMapper.java index 40626c756..4537c978f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatConfigMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatConfigMapper.java @@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.server.persistence.mapper; import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO; -import java.util.List; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + @Mapper public interface ChatConfigMapper { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMapper.java index 6774373d3..a5f21ebd4 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMapper.java @@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.server.persistence.mapper; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO; -import java.util.List; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + @Mapper public interface ChatMapper { @@ -21,5 +22,4 @@ public interface ChatMapper { boolean updateFeedback(QueryDO queryDO); Boolean deleteChat(Long chatId, String userName); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMemoryMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMemoryMapper.java index b9be150f4..6f4341c82 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMemoryMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatMemoryMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ChatMemoryMapper extends BaseMapper { - -} \ No newline at end of file +public interface ChatMemoryMapper extends BaseMapper {} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatParseMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatParseMapper.java index dd2c3648b..c733c5abb 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatParseMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatParseMapper.java @@ -6,7 +6,6 @@ import org.apache.ibatis.annotations.Param; import java.util.List; - @Mapper public interface ChatParseMapper { @@ -19,5 +18,4 @@ public interface ChatParseMapper { List getParseInfoList(List questionIds); List getContextualParseInfo(Integer chatId); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatQueryDOMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatQueryDOMapper.java index 3aac3d9f0..95ccc4ae9 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatQueryDOMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatQueryDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ChatQueryDOMapper extends BaseMapper { - -} +public interface ChatQueryDOMapper extends BaseMapper {} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java index 3f0769904..21b675d77 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface PluginDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface PluginDOMapper extends BaseMapper {} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/custom/ShowCaseCustomMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/custom/ShowCaseCustomMapper.java index 05eb77347..864fa0391 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/custom/ShowCaseCustomMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/custom/ShowCaseCustomMapper.java @@ -2,11 +2,11 @@ package com.tencent.supersonic.chat.server.persistence.mapper.custom; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import org.apache.ibatis.annotations.Mapper; + import java.util.List; @Mapper public interface ShowCaseCustomMapper { List queryShowCase(int start, int limit, int agentId, String userName); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatConfigRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatConfigRepository.java index 6d5deed21..8f98695bb 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatConfigRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatConfigRepository.java @@ -1,9 +1,8 @@ package com.tencent.supersonic.chat.server.persistence.repository; - -import com.tencent.supersonic.chat.server.config.ChatConfig; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.server.config.ChatConfig; import java.util.List; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java index c57d75ad7..9884a2492 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java @@ -7,5 +7,4 @@ public interface ChatContextRepository { ChatContext getOrCreateContext(Integer chatId); void updateContext(ChatContext chatCtx); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java index 51099181d..0b18d5a80 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java @@ -30,11 +30,12 @@ public interface ChatQueryRepository { Long createChatQuery(ChatParseReq chatParseReq); - List batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult, - List candidateParses); + List batchSaveParseInfo( + ChatParseReq chatParseReq, + ParseResp parseResult, + List candidateParses); ChatParseDO getParseInfo(Long questionId, int parseId); List getParseInfoList(List questionIds); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatRepository.java index 945f19928..309d7d285 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatRepository.java @@ -1,7 +1,8 @@ package com.tencent.supersonic.chat.server.persistence.repository; -import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; +import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO; + import java.util.List; public interface ChatRepository { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatConfigRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatConfigRepositoryImpl.java index 047cca4b6..b91db2332 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatConfigRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatConfigRepositoryImpl.java @@ -1,21 +1,21 @@ package com.tencent.supersonic.chat.server.persistence.repository.impl; -import com.tencent.supersonic.chat.server.config.ChatConfig; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; -import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.server.config.ChatConfig; +import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO; import com.tencent.supersonic.chat.server.persistence.mapper.ChatConfigMapper; -import com.tencent.supersonic.chat.server.util.ChatConfigHelper; import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository; - -import java.util.ArrayList; -import java.util.List; +import com.tencent.supersonic.chat.server.util.ChatConfigHelper; import org.springframework.beans.BeanUtils; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Repository; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; + @Repository @Primary public class ChatConfigRepositoryImpl implements ChatConfigRepository { @@ -23,8 +23,8 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository { private final ChatConfigHelper chatConfigHelper; private final ChatConfigMapper chatConfigMapper; - public ChatConfigRepositoryImpl(ChatConfigHelper chatConfigHelper, - ChatConfigMapper chatConfigMapper) { + public ChatConfigRepositoryImpl( + ChatConfigHelper chatConfigHelper, ChatConfigMapper chatConfigMapper) { this.chatConfigHelper = chatConfigHelper; this.chatConfigMapper = chatConfigMapper; } @@ -41,7 +41,6 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository { ChatConfigDO chaConfigDO = chatConfigHelper.chatConfig2DO(chaConfig); return chatConfigMapper.editConfig(chaConfigDO); - } @Override @@ -52,9 +51,12 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository { filterInternal.setStatus(filter.getStatus().getCode()); List chaConfigDOList = chatConfigMapper.search(filterInternal); if (!CollectionUtils.isEmpty(chaConfigDOList)) { - chaConfigDOList.stream().forEach(chaConfigDO -> - chaConfigDescriptorList.add(chatConfigHelper - .chatConfigDO2Descriptor(chaConfigDO.getModelId(), chaConfigDO))); + chaConfigDOList.stream() + .forEach( + chaConfigDO -> + chaConfigDescriptorList.add( + chatConfigHelper.chatConfigDO2Descriptor( + chaConfigDO.getModelId(), chaConfigDO))); } return chaConfigDescriptorList; } @@ -64,5 +66,4 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository { ChatConfigDO chaConfigPO = chatConfigMapper.fetchConfigByModelId(modelId); return chatConfigHelper.chatConfigDO2Descriptor(modelId, chaConfigPO); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java index 7a7491491..c31c354bf 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.chat.server.persistence.repository.impl; import com.google.gson.Gson; +import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO; +import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper; import com.tencent.supersonic.chat.server.persistence.repository.ChatContextRepository; import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO; -import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Repository; @@ -16,7 +16,6 @@ import org.springframework.stereotype.Repository; @Slf4j public class ChatContextRepositoryImpl implements ChatContextRepository { - private final ChatContextMapper chatContextMapper; public ChatContextRepositoryImpl(ChatContextMapper chatContextMapper) { @@ -50,8 +49,8 @@ public class ChatContextRepositoryImpl implements ChatContextRepository { chatContext.setUser(contextDO.getUser()); chatContext.setQueryText(contextDO.getQueryText()); if (contextDO.getSemanticParse() != null && !contextDO.getSemanticParse().isEmpty()) { - SemanticParseInfo semanticParseInfo = JsonUtil.toObject(contextDO.getSemanticParse(), - SemanticParseInfo.class); + SemanticParseInfo semanticParseInfo = + JsonUtil.toObject(contextDO.getSemanticParse(), SemanticParseInfo.class); chatContext.setParseInfo(semanticParseInfo); } return chatContext; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatMemoryRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatMemoryRepositoryImpl.java index 6d71bcd5d..25ce1655c 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatMemoryRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatMemoryRepositoryImpl.java @@ -38,5 +38,4 @@ public class ChatMemoryRepositoryImpl implements ChatMemoryRepository { public List getMemories(QueryWrapper queryWrapper) { return chatMemoryMapper.selectList(queryWrapper); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java index 85e350bd5..639e42209 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java @@ -7,6 +7,8 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; @@ -18,9 +20,7 @@ import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.PageUtils; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -40,14 +40,11 @@ import java.util.stream.Collectors; @Slf4j public class ChatQueryRepositoryImpl implements ChatQueryRepository { - @Autowired - private ChatQueryDOMapper chatQueryDOMapper; + @Autowired private ChatQueryDOMapper chatQueryDOMapper; - @Autowired - private ChatParseMapper chatParseMapper; + @Autowired private ChatParseMapper chatParseMapper; - @Autowired - private ShowCaseCustomMapper showCaseCustomMapper; + @Autowired private ShowCaseCustomMapper showCaseCustomMapper; @Override public PageInfo getChatQuery(PageQueryInfoReq pageQueryInfoReq, Long chatId) { @@ -65,9 +62,9 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { queryWrapper.lambda().ne(ChatQueryDO::getQueryResult, ""); queryWrapper.lambda().orderByDesc(ChatQueryDO::getQuestionId); - PageInfo pageInfo = PageHelper.startPage(pageQueryInfoReq.getCurrent(), - pageQueryInfoReq.getPageSize()) - .doSelectPageInfo(() -> chatQueryDOMapper.selectList(queryWrapper)); + PageInfo pageInfo = + PageHelper.startPage(pageQueryInfoReq.getCurrent(), pageQueryInfoReq.getPageSize()) + .doSelectPageInfo(() -> chatQueryDOMapper.selectList(queryWrapper)); PageInfo chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo); chatQueryVOPageInfo.setList( @@ -104,24 +101,31 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { @Override public List queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId) { - return showCaseCustomMapper.queryShowCase(pageQueryInfoReq.getLimitStart(), - pageQueryInfoReq.getPageSize(), agentId, pageQueryInfoReq.getUserName()) - .stream().map(this::convertTo) + return showCaseCustomMapper + .queryShowCase( + pageQueryInfoReq.getLimitStart(), + pageQueryInfoReq.getPageSize(), + agentId, + pageQueryInfoReq.getUserName()) + .stream() + .map(this::convertTo) .collect(Collectors.toList()); } private QueryResp convertTo(ChatQueryDO chatQueryDO) { QueryResp queryResp = new QueryResp(); BeanUtils.copyProperties(chatQueryDO, queryResp); - QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class); + QueryResult queryResult = + JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class); if (queryResult != null) { queryResult.setQueryId(chatQueryDO.getQuestionId()); queryResp.setQueryResult(queryResult); } - queryResp.setSimilarQueries(JSONObject.parseArray(chatQueryDO.getSimilarQueries(), - SimilarQueryRecallResp.class)); - queryResp.setParseTimeCost(JsonUtil.toObject(chatQueryDO.getParseTimeCost(), - ParseTimeCostResp.class)); + queryResp.setSimilarQueries( + JSONObject.parseArray( + chatQueryDO.getSimilarQueries(), SimilarQueryRecallResp.class)); + queryResp.setParseTimeCost( + JsonUtil.toObject(chatQueryDO.getParseTimeCost(), ParseTimeCostResp.class)); return queryResp; } @@ -143,8 +147,10 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } @Override - public List batchSaveParseInfo(ChatParseReq chatParseReq, - ParseResp parseResult, List candidateParses) { + public List batchSaveParseInfo( + ChatParseReq chatParseReq, + ParseResp parseResult, + List candidateParses) { List chatParseDOList = new ArrayList<>(); getChatParseDO(chatParseReq, parseResult.getQueryId(), candidateParses, chatParseDOList); if (!CollectionUtils.isEmpty(candidateParses)) { @@ -153,8 +159,11 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { return chatParseDOList; } - public void getChatParseDO(ChatParseReq chatParseReq, Long queryId, - List parses, List chatParseDOList) { + public void getChatParseDO( + ChatParseReq chatParseReq, + Long queryId, + List parses, + List chatParseDOList) { for (int i = 0; i < parses.size(); i++) { ChatParseDO chatParseDO = new ChatParseDO(); chatParseDO.setChatId(chatParseReq.getChatId()); @@ -190,5 +199,4 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { public List getParseInfoList(List questionIds) { return chatParseMapper.getParseInfoList(questionIds); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatRepositoryImpl.java index 6b3bf3697..424a4349a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatRepositoryImpl.java @@ -56,5 +56,4 @@ public class ChatRepositoryImpl implements ChatRepository { public Boolean deleteChat(Long chatId, String userName) { return chatMapper.deleteChat(chatId, userName); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java index dd161d387..2303dfe44 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java @@ -77,5 +77,4 @@ public class PluginRepositoryImpl implements PluginRepository { public void deletePlugin(Long id) { pluginDOMapper.deleteById(id); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ChatPlugin.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ChatPlugin.java index b54f49a30..d33aca860 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ChatPlugin.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ChatPlugin.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.chat.server.plugin; - import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; + import java.util.List; @Data @@ -14,37 +14,30 @@ public class ChatPlugin extends RecordInfo { private Long id; - /*** - * plugin type WEB_PAGE WEB_SERVICE - */ + /** * plugin type WEB_PAGE WEB_SERVICE */ private String type; private List dataSetList = Lists.newArrayList(); - /** - * description, for parsing - */ + /** description, for parsing */ private String pattern; - /** - * parse - */ + /** parse */ private ParseMode parseMode; private String parseModeConfig; private String name; - /** - * config for different plugin type - */ + /** config for different plugin type */ private String config; private String comment; public List getExampleQuestionList() { if (StringUtils.isNotBlank(parseModeConfig)) { - PluginParseConfig pluginParseConfig = JSONObject.parseObject(parseModeConfig, PluginParseConfig.class); + PluginParseConfig pluginParseConfig = + JSONObject.parseObject(parseModeConfig, PluginParseConfig.class); return pluginParseConfig.getExamples(); } return Lists.newArrayList(); @@ -57,5 +50,4 @@ public class ChatPlugin extends RecordInfo { public Long getDefaultMode() { return -1L; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java index 628cf4904..6e5a7b393 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java @@ -1,8 +1,6 @@ package com.tencent.supersonic.chat.server.plugin; public enum ParseMode { - EMBEDDING_RECALL, FUNCTION_CALL; - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java index 7d41238f8..19c92c388 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java @@ -46,11 +46,9 @@ import java.util.stream.Collectors; @Component public class PluginManager { - @Autowired - private EmbeddingConfig embeddingConfig; + @Autowired private EmbeddingConfig embeddingConfig; - @Autowired - private EmbeddingService embeddingService; + @Autowired private EmbeddingService embeddingService; public static List getPluginAgentCanSupport(ParseContext parseContext) { PluginService pluginService = ContextUtils.getBean(PluginService.class); @@ -59,14 +57,21 @@ public class PluginManager { if (Objects.isNull(agent)) { return plugins; } - List pluginIds = getPluginTools(agent).stream().map(PluginTool::getPlugins) - .flatMap(Collection::stream).collect(Collectors.toList()); + List pluginIds = + getPluginTools(agent).stream() + .map(PluginTool::getPlugins) + .flatMap(Collection::stream) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(pluginIds)) { return Lists.newArrayList(); } - plugins = plugins.stream().filter(plugin -> pluginIds.contains(plugin.getId())) - .collect(Collectors.toList()); - log.info("plugins witch can be supported by cur agent :{} {}", agent.getName(), + plugins = + plugins.stream() + .filter(plugin -> pluginIds.contains(plugin.getId())) + .collect(Collectors.toList()); + log.info( + "plugins witch can be supported by cur agent :{} {}", + agent.getName(), plugins.stream().map(ChatPlugin::getName).collect(Collectors.toList())); return plugins; } @@ -79,7 +84,8 @@ public class PluginManager { if (CollectionUtils.isEmpty(tools)) { return Lists.newArrayList(); } - return tools.stream().map(tool -> JSONObject.parseObject(tool, PluginTool.class)) + return tools.stream() + .map(tool -> JSONObject.parseObject(tool, PluginTool.class)) .collect(Collectors.toList()); } @@ -136,18 +142,23 @@ public class PluginManager { public RetrieveQueryResult recognize(String embeddingText) { - RetrieveQuery retrieveQuery = RetrieveQuery.builder() - .queryTextsList(Collections.singletonList(embeddingText)) - .build(); + RetrieveQuery retrieveQuery = + RetrieveQuery.builder() + .queryTextsList(Collections.singletonList(embeddingText)) + .build(); - List resultList = embeddingService.retrieveQuery(embeddingConfig.getPresetCollection(), - retrieveQuery, embeddingConfig.getNResult()); + List resultList = + embeddingService.retrieveQuery( + embeddingConfig.getPresetCollection(), + retrieveQuery, + embeddingConfig.getNResult()); if (CollectionUtils.isNotEmpty(resultList)) { for (RetrieveQueryResult embeddingResp : resultList) { List embeddingRetrievals = embeddingResp.getRetrieval(); for (Retrieval embeddingRetrieval : embeddingRetrievals) { - embeddingRetrieval.setId(getPluginIdFromEmbeddingId(embeddingRetrieval.getId())); + embeddingRetrieval.setId( + getPluginIdFromEmbeddingId(embeddingRetrieval.getId())); } } return resultList.get(0); @@ -162,7 +173,8 @@ public class PluginManager { int num = 0; for (String pattern : exampleQuestions) { TextSegment query = TextSegment.from(pattern); - TextSegmentConvert.addQueryId(query, generateUniqueEmbeddingId(num, plugin.getId())); + TextSegmentConvert.addQueryId( + query, generateUniqueEmbeddingId(num, plugin.getId())); queries.add(query); num++; } @@ -178,7 +190,7 @@ public class PluginManager { return embeddingIdSet; } - //num can not bigger than 100 + // num can not bigger than 100 private String generateUniqueEmbeddingId(int num, Long pluginId) { if (num < 10) { return String.format("%s00%s", pluginId, num); @@ -202,8 +214,8 @@ public class PluginManager { return Pair.of(true, pluginMatchedDataSet); } Set matchedDataSet = Sets.newHashSet(); - Map> paramOptionMap = paramOptions.stream() - .collect(Collectors.groupingBy(ParamOption::getDataSetId)); + Map> paramOptionMap = + paramOptions.stream().collect(Collectors.groupingBy(ParamOption::getDataSetId)); for (Long dataSetId : paramOptionMap.keySet()) { List params = paramOptionMap.get(dataSetId); if (CollectionUtils.isEmpty(params)) { @@ -237,9 +249,13 @@ public class PluginManager { if (org.springframework.util.CollectionUtils.isEmpty(schemaElementMatches)) { return Sets.newHashSet(); } - return schemaElementMatches.stream().filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) - || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) + return schemaElementMatches.stream() + .filter( + schemaElementMatch -> + SchemaElementType.VALUE.equals( + schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals( + schemaElementMatch.getElement().getType())) .map(SchemaElementMatch::getElement) .map(SchemaElement::getId) .collect(Collectors.toSet()); @@ -255,7 +271,9 @@ public class PluginManager { return Lists.newArrayList(); } return paramOptions.stream() - .filter(paramOption -> ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType())) + .filter( + paramOption -> + ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType())) .collect(Collectors.toList()); } @@ -273,5 +291,4 @@ public class PluginManager { } return pluginMatchedDataSet; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java index ace38a4ba..2dd671148 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.plugin; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -22,5 +21,4 @@ public class PluginParseConfig implements Serializable { private String name; private String description; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginQueryManager.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginQueryManager.java index 4b649b647..7e3099c70 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginQueryManager.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginQueryManager.java @@ -20,5 +20,4 @@ public class PluginQueryManager { public static PluginSemanticQuery getPluginQuery(String queryMode) { return pluginQueries.get(queryMode); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java index 906f71d92..0f39c1c8b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java @@ -20,5 +20,4 @@ public class PluginRecallResult { private double score; private double distance; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java index a695d0120..580254077 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java @@ -22,16 +22,17 @@ public class ParamOption { private Object value; /** - * CUSTOM: the value is specified by the user - * SEMANTIC: the value of element - * FORWARD: only forward + * CUSTOM: the value is specified by the user SEMANTIC: the value of element FORWARD: only + * forward */ public enum ParamType { - CUSTOM, SEMANTIC, FORWARD + CUSTOM, + SEMANTIC, + FORWARD } public enum OptionType { - REQUIRED, OPTIONAL + REQUIRED, + OPTIONAL } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java index f9e5ff98e..df5b4c9e8 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.chat.server.plugin.build; import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.plugin.PluginParseResult; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -43,29 +43,40 @@ public abstract class PluginSemanticQuery { protected Map getElementMap(PluginParseResult pluginParseResult) { Map elementValueMap = new HashMap<>(); Map filterValueMap = getFilterMap(pluginParseResult); - List schemaElementMatchList = parseInfo.getElementMatches() - .stream().filter(schemaElementMatch -> schemaElementMatch.getFrequency() != null) - .sorted(Comparator.comparingLong(SchemaElementMatch::getFrequency).reversed()) - .collect(Collectors.toList()); + List schemaElementMatchList = + parseInfo.getElementMatches().stream() + .filter(schemaElementMatch -> schemaElementMatch.getFrequency() != null) + .sorted( + Comparator.comparingLong(SchemaElementMatch::getFrequency) + .reversed()) + .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(schemaElementMatchList)) { - schemaElementMatchList.stream().filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) - || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) + schemaElementMatchList.stream() + .filter( + schemaElementMatch -> + SchemaElementType.VALUE.equals( + schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals( + schemaElementMatch.getElement().getType())) .filter(schemaElementMatch -> schemaElementMatch.getSimilarity() == 1.0) - .forEach(schemaElementMatch -> { - Object queryFilterValue = filterValueMap.get(schemaElementMatch.getElement().getId()); - if (queryFilterValue != null) { - if (String.valueOf(queryFilterValue).equals(String.valueOf(schemaElementMatch.getWord()))) { - elementValueMap.put( - String.valueOf(schemaElementMatch.getElement().getId()), - schemaElementMatch.getWord()); - } - } else { - elementValueMap.computeIfAbsent( - String.valueOf(schemaElementMatch.getElement().getId()), - k -> schemaElementMatch.getWord()); - } - }); + .forEach( + schemaElementMatch -> { + Object queryFilterValue = + filterValueMap.get(schemaElementMatch.getElement().getId()); + if (queryFilterValue != null) { + if (String.valueOf(queryFilterValue) + .equals(String.valueOf(schemaElementMatch.getWord()))) { + elementValueMap.put( + String.valueOf( + schemaElementMatch.getElement().getId()), + schemaElementMatch.getWord()); + } + } else { + elementValueMap.computeIfAbsent( + String.valueOf(schemaElementMatch.getElement().getId()), + k -> schemaElementMatch.getWord()); + } + }); } return elementValueMap; } @@ -97,5 +108,4 @@ public abstract class PluginSemanticQuery { public void setParseInfo(SemanticParseInfo parseInfo) { this.parseInfo = parseInfo; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java index 86e65201d..548b588ab 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java @@ -15,5 +15,4 @@ public class WebBase { public List getParams() { return paramOptions; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java index 90f2fca01..7af00cad6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.chat.server.plugin.build.webpage; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.plugin.ChatPlugin; import com.tencent.supersonic.chat.server.plugin.PluginParseResult; import com.tencent.supersonic.chat.server.plugin.PluginQueryManager; @@ -7,7 +8,6 @@ import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery; import com.tencent.supersonic.chat.server.plugin.build.WebBase; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -41,12 +41,13 @@ public class WebPageQuery extends PluginSemanticQuery { QueryResult queryResult = new QueryResult(); queryResult.setQueryMode(QUERY_MODE); Map properties = parseInfo.getProperties(); - PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)), - PluginParseResult.class); + PluginParseResult pluginParseResult = + JsonUtil.toObject( + JsonUtil.toString(properties.get(Constants.CONTEXT)), + PluginParseResult.class); WebPageResp webPageResponse = buildResponse(pluginParseResult); queryResult.setResponse(webPageResponse); queryResult.setQueryState(QueryState.SUCCESS); return queryResult; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java index 67db6508d..3881ee825 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java @@ -1,12 +1,10 @@ package com.tencent.supersonic.chat.server.plugin.build.webpage; - import com.tencent.supersonic.chat.server.plugin.build.WebBase; import lombok.Data; import java.util.List; - @Data public class WebPageResp { @@ -21,5 +19,4 @@ public class WebPageResp { private WebBase webPage; private List moreWebPage; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java index c1fde5203..87c57f5bc 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.server.plugin.build.webservice; import com.alibaba.fastjson.JSON; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.plugin.ChatPlugin; import com.tencent.supersonic.chat.server.plugin.PluginParseResult; import com.tencent.supersonic.chat.server.plugin.PluginQueryManager; @@ -11,7 +12,6 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - @Slf4j @Component public class WebServiceQuery extends PluginSemanticQuery { @@ -46,15 +45,18 @@ public class WebServiceQuery extends PluginSemanticQuery { QueryResult queryResult = new QueryResult(); queryResult.setQueryMode(QUERY_MODE); Map properties = parseInfo.getProperties(); - PluginParseResult pluginParseResult = JsonUtil.toObject( - JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class); + PluginParseResult pluginParseResult = + JsonUtil.toObject( + JsonUtil.toString(properties.get(Constants.CONTEXT)), + PluginParseResult.class); WebServiceResp webServiceResponse = buildResponse(pluginParseResult); Object object = webServiceResponse.getResult(); // in order to show webServiceQuery result int frontend conveniently, // webServiceResponse result format is consistent with queryByStruct result. log.info("webServiceResponse result:{}", JsonUtil.toString(object)); try { - Map data = JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class); + Map data = + JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class); if (data.get("resultList") != null) { queryResult.setQueryResults((List>) data.get("resultList")); } @@ -72,7 +74,9 @@ public class WebServiceQuery extends PluginSemanticQuery { protected WebServiceResp buildResponse(PluginParseResult pluginParseResult) { WebServiceResp webServiceResponse = new WebServiceResp(); ChatPlugin plugin = pluginParseResult.getPlugin(); - WebBase webBase = fillWebBaseResult(JsonUtil.toObject(plugin.getConfig(), WebBase.class), pluginParseResult); + WebBase webBase = + fillWebBaseResult( + JsonUtil.toObject(plugin.getConfig(), WebBase.class), pluginParseResult); webServiceResponse.setWebBase(webBase); List paramOptions = webBase.getParamOptions(); Map params = new HashMap<>(); @@ -86,7 +90,8 @@ public class WebServiceQuery extends PluginSemanticQuery { Object objectResponse = null; restTemplate = ContextUtils.getBean(RestTemplate.class); try { - responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class); + responseEntity = + restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class); objectResponse = responseEntity.getBody(); log.info("objectResponse:{}", objectResponse); Map response = JsonUtil.objectToMap(objectResponse); @@ -96,5 +101,4 @@ public class WebServiceQuery extends PluginSemanticQuery { } return webServiceResponse; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java index 02744324e..af21d3e3d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java @@ -1,15 +1,12 @@ package com.tencent.supersonic.chat.server.plugin.build.webservice; - import com.tencent.supersonic.chat.server.plugin.build.WebBase; import lombok.Data; - @Data public class WebServiceResp { private WebBase webBase; private Object result; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java index e68c961d7..5f1aaee99 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.plugin.event; - import com.tencent.supersonic.chat.server.plugin.ChatPlugin; import org.springframework.context.ApplicationEvent; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java index c2bc6b785..a74116c02 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java @@ -22,5 +22,4 @@ public class PluginUpdateEvent extends ApplicationEvent { public ChatPlugin getNewPlugin() { return newPlugin; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/PluginRecognizer.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/PluginRecognizer.java index e52f9227a..74dde3215 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/PluginRecognizer.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/PluginRecognizer.java @@ -23,9 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -/** - * PluginParser defines the basic process and common methods for recalling plugins. - */ +/** PluginParser defines the basic process and common methods for recalling plugins. */ public abstract class PluginRecognizer { public void recognize(ParseContext parseContext, ParseResp parseResp) { @@ -43,16 +41,17 @@ public abstract class PluginRecognizer { public abstract PluginRecallResult recallPlugin(ParseContext parseContext); - public void buildQuery(ParseContext parseContext, ParseResp parseResp, - PluginRecallResult pluginRecallResult) { + public void buildQuery( + ParseContext parseContext, ParseResp parseResp, PluginRecallResult pluginRecallResult) { ChatPlugin plugin = pluginRecallResult.getPlugin(); Set dataSetIds = pluginRecallResult.getDataSetIds(); if (plugin.isContainsAllDataSet()) { dataSetIds = Sets.newHashSet(-1L); } for (Long dataSetId : dataSetIds) { - SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(dataSetId, plugin, - parseContext, pluginRecallResult.getDistance()); + SemanticParseInfo semanticParseInfo = + buildSemanticParseInfo( + dataSetId, plugin, parseContext, pluginRecallResult.getDistance()); semanticParseInfo.setQueryMode(plugin.getType()); semanticParseInfo.setScore(pluginRecallResult.getScore()); parseResp.getSelectedParses().add(semanticParseInfo); @@ -63,9 +62,10 @@ public abstract class PluginRecognizer { return PluginManager.getPluginAgentCanSupport(parseContext); } - protected SemanticParseInfo buildSemanticParseInfo(Long dataSetId, ChatPlugin plugin, - ParseContext parseContext, double distance) { - List schemaElementMatches = parseContext.getMapInfo().getMatchedElements(dataSetId); + protected SemanticParseInfo buildSemanticParseInfo( + Long dataSetId, ChatPlugin plugin, ParseContext parseContext, double distance) { + List schemaElementMatches = + parseContext.getMapInfo().getMatchedElements(dataSetId); QueryFilters queryFilters = parseContext.getQueryFilters(); if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); @@ -96,18 +96,22 @@ public abstract class PluginRecognizer { if (CollectionUtils.isEmpty(schemaElementMatches)) { return; } - schemaElementMatches.stream().filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) - || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) - .forEach(schemaElementMatch -> { - QueryFilter queryFilter = new QueryFilter(); - queryFilter.setValue(schemaElementMatch.getWord()); - queryFilter.setElementID(schemaElementMatch.getElement().getId()); - queryFilter.setName(schemaElementMatch.getElement().getName()); - queryFilter.setOperator(FilterOperatorEnum.EQUALS); - queryFilter.setBizName(schemaElementMatch.getElement().getBizName()); - semanticParseInfo.getDimensionFilters().add(queryFilter); - }); + schemaElementMatches.stream() + .filter( + schemaElementMatch -> + SchemaElementType.VALUE.equals( + schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals( + schemaElementMatch.getElement().getType())) + .forEach( + schemaElementMatch -> { + QueryFilter queryFilter = new QueryFilter(); + queryFilter.setValue(schemaElementMatch.getWord()); + queryFilter.setElementID(schemaElementMatch.getElement().getId()); + queryFilter.setName(schemaElementMatch.getElement().getName()); + queryFilter.setOperator(FilterOperatorEnum.EQUALS); + queryFilter.setBizName(schemaElementMatch.getElement().getBizName()); + semanticParseInfo.getDimensionFilters().add(queryFilter); + }); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/EmbeddingRecallRecognizer.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/EmbeddingRecallRecognizer.java index 86c7bbb37..a32318fdf 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/EmbeddingRecallRecognizer.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/EmbeddingRecallRecognizer.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.plugin.recognize.embedding; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.server.plugin.ParseMode; import com.tencent.supersonic.chat.server.plugin.ChatPlugin; +import com.tencent.supersonic.chat.server.plugin.ParseMode; import com.tencent.supersonic.chat.server.plugin.PluginManager; import com.tencent.supersonic.chat.server.plugin.PluginRecallResult; import com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer; @@ -20,9 +20,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -/** - * EmbeddingRecallParser is an implementation of a recall plugin based on Embedding - */ +/** EmbeddingRecallParser is an implementation of a recall plugin based on Embedding */ @Slf4j public class EmbeddingRecallRecognizer extends PluginRecognizer { @@ -38,7 +36,8 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer { return null; } List plugins = getPluginList(parseContext); - Map pluginMap = plugins.stream().collect(Collectors.toMap(ChatPlugin::getId, p -> p)); + Map pluginMap = + plugins.stream().collect(Collectors.toMap(ChatPlugin::getId, p -> p)); for (Retrieval embeddingRetrieval : embeddingRetrievals) { ChatPlugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId())); if (plugin == null) { @@ -55,7 +54,11 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer { double distance = embeddingRetrieval.getDistance(); double score = parseContext.getQueryText().length() * (1 - distance); return PluginRecallResult.builder() - .plugin(plugin).dataSetIds(dataSetList).score(score).distance(distance).build(); + .plugin(plugin) + .dataSetIds(dataSetList) + .score(score) + .distance(distance) + .build(); } } return null; @@ -68,8 +71,10 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer { List embeddingRetrievals = embeddingResp.getRetrieval(); if (!CollectionUtils.isEmpty(embeddingRetrievals)) { - embeddingRetrievals = embeddingRetrievals.stream().sorted(Comparator.comparingDouble(o -> - Math.abs(o.getDistance()))).collect(Collectors.toList()); + embeddingRetrievals = + embeddingRetrievals.stream() + .sorted(Comparator.comparingDouble(o -> Math.abs(o.getDistance()))) + .collect(Collectors.toList()); embeddingResp.setRetrieval(embeddingRetrievals); } return embeddingRetrievals; @@ -78,5 +83,4 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer { } return Lists.newArrayList(); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrieval.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrieval.java index 02a808051..8898a3f8f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrieval.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrieval.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.plugin.recognize.embedding; - import lombok.Data; @Data @@ -15,5 +14,4 @@ public class RecallRetrieval { private String presetId; private String query; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrievalResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrievalResp.java index ae714b337..324823d88 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrievalResp.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/RecallRetrievalResp.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.plugin.recognize.embedding; - import lombok.Data; + import java.util.List; @Data @@ -10,6 +10,4 @@ public class RecallRetrievalResp { private String query; private List retrieval; - - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatContext.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatContext.java index 955028d4e..6d416565e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatContext.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatContext.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.pojo; - import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Data; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java index 19e39704b..5075e863c 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java @@ -1,8 +1,4 @@ package com.tencent.supersonic.chat.server.processor; -/** - * A ResultProcessor wraps things up before returning results to users. - */ -public interface ResultProcessor { - -} +/** A ResultProcessor wraps things up before returning results to users. */ +public interface ResultProcessor {} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index f30a76e2b..19cabdcfd 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.pojo.ExecuteContext; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; @@ -8,7 +9,6 @@ import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import org.springframework.util.CollectionUtils; @@ -20,8 +20,7 @@ import java.util.Set; import java.util.stream.Collectors; /** - * DimensionRecommendProcessor recommend some dimensions - * related to metrics based on configuration + * DimensionRecommendProcessor recommend some dimensions related to metrics based on configuration */ public class DimensionRecommendProcessor implements ExecuteResultProcessor { @@ -50,13 +49,20 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { List drillDownDimensions = Lists.newArrayList(); Set metricElements = dataSetSchema.getMetrics(); if (!CollectionUtils.isEmpty(metricElements)) { - Optional metric = metricElements.stream().filter(schemaElement -> - metricId.equals(schemaElement.getId()) - && !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements())) - .findFirst(); + Optional metric = + metricElements.stream() + .filter( + schemaElement -> + metricId.equals(schemaElement.getId()) + && !CollectionUtils.isEmpty( + schemaElement + .getRelatedSchemaElements())) + .findFirst(); if (metric.isPresent()) { - drillDownDimensions = metric.get().getRelatedSchemaElements().stream() - .map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList()); + drillDownDimensions = + metric.get().getRelatedSchemaElements().stream() + .map(RelatedSchemaElement::getDimensionId) + .collect(Collectors.toList()); } } final List drillDownDimensionsFinal = drillDownDimensions; @@ -76,5 +82,4 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { } return Objects.nonNull(dimension.getUseCnt()); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java index 23541320e..a72fb8f85 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java @@ -1,15 +1,11 @@ package com.tencent.supersonic.chat.server.processor.execute; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.pojo.ExecuteContext; import com.tencent.supersonic.chat.server.processor.ResultProcessor; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -/** - * A ExecuteResultProcessor wraps things up before returning - * execution results to the users. - */ +/** A ExecuteResultProcessor wraps things up before returning execution results to the users. */ public interface ExecuteResultProcessor extends ResultProcessor { void process(ExecuteContext executeContext, QueryResult queryResult); - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java index e2fc01cea..a5f669627 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java @@ -1,17 +1,9 @@ package com.tencent.supersonic.chat.server.processor.execute; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.pojo.ExecuteContext; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.QueryColumn; @@ -20,17 +12,19 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.RatioOverType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.AggregateInfo; import com.tencent.supersonic.headless.api.pojo.MetricInfo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.core.config.AggregatorConfig; import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder; +import com.tencent.supersonic.headless.core.config.AggregatorConfig; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.text.DecimalFormat; import java.time.DayOfWeek; import java.time.LocalDate; @@ -49,13 +43,17 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; -/** - * Add ratio queries for metric queries. - */ +/** Add ratio queries for metric queries. */ @Slf4j public class MetricRatioProcessor implements ExecuteResultProcessor { @@ -68,18 +66,24 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { || !QueryType.METRIC.equals(semanticParseInfo.getQueryType())) { return; } - AggregateInfo aggregateInfo = getAggregateInfo(executeContext.getUser(), - semanticParseInfo, queryResult); + AggregateInfo aggregateInfo = + getAggregateInfo(executeContext.getUser(), semanticParseInfo, queryResult); queryResult.setAggregateInfo(aggregateInfo); } - public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) { + public AggregateInfo getAggregateInfo( + User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) { Set resultMetricNames = new HashSet<>(); - queryResult.getQueryColumns() - .stream().forEach(c -> resultMetricNames.addAll(SqlSelectHelper.getColumnFromExpr(c.getNameEn()))); - Optional ratioMetric = semanticParseInfo.getMetrics().stream() - .filter(m -> resultMetricNames.contains(m.getBizName())).findFirst(); + queryResult.getQueryColumns().stream() + .forEach( + c -> + resultMetricNames.addAll( + SqlSelectHelper.getColumnFromExpr(c.getNameEn()))); + Optional ratioMetric = + semanticParseInfo.getMetrics().stream() + .filter(m -> resultMetricNames.contains(m.getBizName())) + .findFirst(); AggregateInfo aggregateInfo = new AggregateInfo(); if (!ratioMetric.isPresent()) { @@ -88,31 +92,48 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { try { String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); - Optional lastDayOp = queryResult.getQueryResults().stream() - .filter(r -> r.containsKey(dateField)) - .map(r -> r.get(dateField).toString()) - .sorted(Comparator.reverseOrder()).findFirst(); + Optional lastDayOp = + queryResult.getQueryResults().stream() + .filter(r -> r.containsKey(dateField)) + .map(r -> r.get(dateField).toString()) + .sorted(Comparator.reverseOrder()) + .findFirst(); if (!lastDayOp.isPresent()) { return new AggregateInfo(); } - Optional> lastValue = queryResult.getQueryResults().stream() - .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst(); + Optional> lastValue = + queryResult.getQueryResults().stream() + .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())) + .findFirst(); MetricInfo metricInfo = new MetricInfo(); metricInfo.setStatistics(new HashMap<>()); - if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) { + if (lastValue.isPresent() + && lastValue.get().containsKey(ratioMetric.get().getBizName())) { DecimalFormat df = new DecimalFormat("#.####"); metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName()))); } metricInfo.setDate(lastValue.get().get(dateField).toString()); - CompletableFuture metricInfoRoll = CompletableFuture.supplyAsync( - () -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL, - queryResult)); - CompletableFuture metricInfoOver = CompletableFuture.supplyAsync( - () -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER, - queryResult)); + CompletableFuture metricInfoRoll = + CompletableFuture.supplyAsync( + () -> + queryRatio( + user, + semanticParseInfo, + ratioMetric.get(), + AggOperatorEnum.RATIO_ROLL, + queryResult)); + CompletableFuture metricInfoOver = + CompletableFuture.supplyAsync( + () -> + queryRatio( + user, + semanticParseInfo, + ratioMetric.get(), + AggOperatorEnum.RATIO_OVER, + queryResult)); CompletableFuture.allOf(metricInfoRoll, metricInfoOver); metricInfo.setName(metricInfoRoll.get().getName()); metricInfo.setValue(metricInfoRoll.get().getValue()); @@ -126,13 +147,19 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { } @SneakyThrows - private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, - AggOperatorEnum aggOperatorEnum, QueryResult queryResult) { + private MetricInfo queryRatio( + User user, + SemanticParseInfo semanticParseInfo, + SchemaElement metric, + AggOperatorEnum aggOperatorEnum, + QueryResult queryResult) { - QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); + QueryStructReq queryStructReq = + QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField))); - queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult)); + queryStructReq.setDateInfo( + getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult)); queryStructReq.setConvertToSql(false); SemanticLayerService queryService = ContextUtils.getBean(SemanticLayerService.class); SemanticQueryResp queryResp = queryService.queryByReq(queryStructReq, user); @@ -143,21 +170,26 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { } Map result = queryResp.getResultList().get(0); - Optional valueColumn = queryResp.getColumns().stream() - .filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst(); + Optional valueColumn = + queryResp.getColumns().stream() + .filter(c -> c.getNameEn().equals(metric.getBizName())) + .findFirst(); if (!valueColumn.isPresent()) { return metricInfo; } - String valueField = String.format("%s_%s", valueColumn.get().getNameEn(), aggOperatorEnum.getOperator()); + String valueField = + String.format( + "%s_%s", valueColumn.get().getNameEn(), aggOperatorEnum.getOperator()); if (result.containsKey(valueColumn.get().getNameEn())) { DecimalFormat df = new DecimalFormat("#.####"); metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn()))); } String ratio = ""; if (Objects.nonNull(result.get(valueField))) { - ratio = String.format("%.2f", - (Double.valueOf(result.get(valueField).toString()) * 100)) + "%"; + ratio = + String.format("%.2f", (Double.valueOf(result.get(valueField).toString()) * 100)) + + "%"; } String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName(); String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName(); @@ -169,19 +201,28 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { statisticsRollName = RatioOverType.WEEK_ON_WEEK.getShowName(); statisticsOverName = RatioOverType.MONTH_ON_WEEK.getShowName(); } - metricInfo.getStatistics().put(aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) - ? statisticsRollName : statisticsOverName, ratio); + metricInfo + .getStatistics() + .put( + aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) + ? statisticsRollName + : statisticsOverName, + ratio); metricInfo.setName(metric.getName()); return metricInfo; } - private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo, + private DateConf getRatioDateConf( + AggOperatorEnum aggOperatorEnum, + SemanticParseInfo semanticParseInfo, QueryResult queryResult) { String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); - Optional lastDayOp = queryResult.getQueryResults() - .stream().map(r -> r.get(dateField).toString()) - .sorted(Comparator.reverseOrder()).findFirst(); + Optional lastDayOp = + queryResult.getQueryResults().stream() + .map(r -> r.get(dateField).toString()) + .sorted(Comparator.reverseOrder()) + .findFirst(); if (!lastDayOp.isPresent()) { return semanticParseInfo.getDateInfo(); @@ -194,29 +235,37 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { dayList.add(lastDay); String start = ""; if (DAY.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { - DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay, - new String[]{DAY_FORMAT, DAY_FORMAT_INT}); + DateTimeFormatter formatter = + DateUtils.getDateFormatter(lastDay, new String[] {DAY_FORMAT, DAY_FORMAT_INT}); LocalDate end = LocalDate.parse(lastDay, formatter); - start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusDays(1).format(formatter) - : end.minusWeeks(1).format(formatter); + start = + aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) + ? end.minusDays(1).format(formatter) + : end.minusWeeks(1).format(formatter); } if (WEEK.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { - DateTimeFormatter formatter = DateUtils.getTimeFormatter(lastDay, - new String[]{TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT}); + DateTimeFormatter formatter = + DateUtils.getTimeFormatter( + lastDay, + new String[] {TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT}); LocalDateTime end = LocalDateTime.parse(lastDay, formatter); - start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusWeeks(1).format(formatter) - : end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter); + start = + aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) + ? end.minusWeeks(1).format(formatter) + : end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter); } if (MONTH.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { - DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay, - new String[]{MONTH_FORMAT, MONTH_FORMAT_INT}); + DateTimeFormatter formatter = + DateUtils.getDateFormatter( + lastDay, new String[] {MONTH_FORMAT, MONTH_FORMAT_INT}); YearMonth end = YearMonth.parse(lastDay, formatter); - start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusMonths(1).format(formatter) - : end.minusYears(1).format(formatter); + start = + aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) + ? end.minusMonths(1).format(formatter) + : end.minusYears(1).format(formatter); } dayList.add(start); dateConf.setDateList(dayList); return dateConf; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java index 3bc9092b7..e40359dac 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java @@ -26,9 +26,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/** - * MetricRecommendProcessor fills recommended metrics based on embedding similarity. - */ +/** MetricRecommendProcessor fills recommended metrics based on embedding similarity. */ public class MetricRecommendProcessor implements ExecuteResultProcessor { private static final int METRIC_RECOMMEND_SIZE = 5; @@ -44,24 +42,36 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { || CollectionUtils.isEmpty(parseInfo.getMetrics())) { return; } - List metricNames = Collections.singletonList(parseInfo.getMetrics().iterator().next().getName()); + List metricNames = + Collections.singletonList(parseInfo.getMetrics().iterator().next().getName()); Map filterCondition = new HashMap<>(); - filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getDataSetId().toString()); + filterCondition.put( + "modelId", parseInfo.getMetrics().iterator().next().getDataSetId().toString()); filterCondition.put("type", SchemaElementType.METRIC.name()); - RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames) - .filterCondition(filterCondition).queryEmbeddings(null).build(); - MetaEmbeddingService metaEmbeddingService = ContextUtils.getBean(MetaEmbeddingService.class); + RetrieveQuery retrieveQuery = + RetrieveQuery.builder() + .queryTextsList(metricNames) + .filterCondition(filterCondition) + .queryEmbeddings(null) + .build(); + MetaEmbeddingService metaEmbeddingService = + ContextUtils.getBean(MetaEmbeddingService.class); List retrieveQueryResults = - metaEmbeddingService.retrieveQuery(retrieveQuery, METRIC_RECOMMEND_SIZE + 1, new HashMap<>(), - new HashSet<>()); + metaEmbeddingService.retrieveQuery( + retrieveQuery, METRIC_RECOMMEND_SIZE + 1, new HashMap<>(), new HashSet<>()); if (CollectionUtils.isEmpty(retrieveQueryResults)) { return; } - List retrievals = retrieveQueryResults.stream() - .flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream()) - .sorted(Comparator.comparingDouble(Retrieval::getDistance)) - .distinct().collect(Collectors.toList()); - Set metricIds = parseInfo.getMetrics().stream().map(SchemaElement::getId).collect(Collectors.toSet()); + List retrievals = + retrieveQueryResults.stream() + .flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream()) + .sorted(Comparator.comparingDouble(Retrieval::getDistance)) + .distinct() + .collect(Collectors.toList()); + Set metricIds = + parseInfo.getMetrics().stream() + .map(SchemaElement::getId) + .collect(Collectors.toSet()); int metricOrder = 0; for (SchemaElement metric : parseInfo.getMetrics()) { metric.setOrder(metricOrder++); @@ -69,15 +79,23 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { for (Retrieval retrieval : retrievals) { if (!metricIds.contains(Retrieval.getLongId(retrieval.getId()))) { if (Objects.nonNull(retrieval.getMetadata().get("id"))) { - String idStr = retrieval.getMetadata().get("id").toString() - .replaceAll(DictWordType.NATURE_SPILT, ""); + String idStr = + retrieval + .getMetadata() + .get("id") + .toString() + .replaceAll(DictWordType.NATURE_SPILT, ""); retrieval.getMetadata().put("id", idStr); } String metaStr = JSONObject.toJSONString(retrieval.getMetadata()); SchemaElement schemaElement = JSONObject.parseObject(metaStr, SchemaElement.class); if (retrieval.getMetadata().containsKey("dataSetId")) { - String dataSetId = retrieval.getMetadata().get("dataSetId").toString() - .replace(Constants.UNDERLINE, ""); + String dataSetId = + retrieval + .getMetadata() + .get("dataSetId") + .toString() + .replace(Constants.UNDERLINE, ""); schemaElement.setDataSetId(Long.parseLong(dataSetId)); } schemaElement.setOrder(++metricOrder); @@ -85,5 +103,4 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { } } } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java index 33bb53d02..0f5fb864b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java @@ -4,12 +4,8 @@ import com.tencent.supersonic.chat.server.pojo.ParseContext; import com.tencent.supersonic.chat.server.processor.ResultProcessor; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -/** - * A ParseResultProcessor wraps things up before returning - * parsing results to the users. - */ +/** A ParseResultProcessor wraps things up before returning parsing results to the users. */ public interface ParseResultProcessor extends ResultProcessor { void process(ParseContext parseContext, ParseResp parseResp); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java index b8d155ad6..50eb73f7e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java @@ -18,9 +18,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -/** - * MetricRecommendProcessor fills recommended query based on embedding similarity. - */ +/** MetricRecommendProcessor fills recommended query based on embedding similarity. */ @Slf4j public class QueryRecommendProcessor implements ParseResultProcessor { @@ -32,8 +30,8 @@ public class QueryRecommendProcessor implements ParseResultProcessor { @SneakyThrows private void doProcess(ParseResp parseResp, ParseContext parseContext) { Long queryId = parseResp.getQueryId(); - List solvedQueries = getSimilarQueries(parseContext.getQueryText(), - parseContext.getAgent().getId()); + List solvedQueries = + getSimilarQueries(parseContext.getQueryText(), parseContext.getAgent().getId()); ChatQueryDO chatQueryDO = getChatQuery(queryId); chatQueryDO.setSimilarQueries(JSONObject.toJSONString(solvedQueries)); updateChatQuery(chatQueryDO); @@ -43,9 +41,14 @@ public class QueryRecommendProcessor implements ParseResultProcessor { ExemplarService exemplarService = ContextUtils.getBean(ExemplarService.class); EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); String memoryCollectionName = embeddingConfig.getMemoryCollectionName(agentId); - List exemplars = exemplarService.recallExemplars(memoryCollectionName, queryText, 5); - return exemplars.stream().map(sqlExemplar -> - SimilarQueryRecallResp.builder().queryText(sqlExemplar.getQuestion()).build()) + List exemplars = + exemplarService.recallExemplars(memoryCollectionName, queryText, 5); + return exemplars.stream() + .map( + sqlExemplar -> + SimilarQueryRecallResp.builder() + .queryText(sqlExemplar.getQuestion()) + .build()) .collect(Collectors.toList()); } @@ -61,5 +64,4 @@ public class QueryRecommendProcessor implements ParseResultProcessor { updateWrapper.set("similar_queries", chatQueryDO.getSimilarQueries()); chatQueryRepository.updateChatQuery(chatQueryDO, updateWrapper); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java index 7fcc2e044..d1854099f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java @@ -1,21 +1,21 @@ package com.tencent.supersonic.chat.server.processor.parse; - import com.tencent.supersonic.chat.server.pojo.ParseContext; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import lombok.extern.slf4j.Slf4j; -/** - * TimeCostProcessor adds time cost of parsing. - **/ +/** TimeCostProcessor adds time cost of parsing. */ @Slf4j public class TimeCostProcessor implements ParseResultProcessor { @Override public void process(ParseContext parseContext, ParseResp parseResp) { long parseStartTime = parseResp.getParseTimeCost().getParseStartTime(); - parseResp.getParseTimeCost().setParseTime( - System.currentTimeMillis() - parseStartTime - parseResp.getParseTimeCost().getSqlTime()); + parseResp + .getParseTimeCost() + .setParseTime( + System.currentTimeMillis() + - parseStartTime + - parseResp.getParseTimeCost().getSqlTime()); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java index 3a95dd763..cedc23462 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.chat.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.server.agent.Agent; @@ -16,8 +19,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; import java.util.Map; @@ -25,21 +26,22 @@ import java.util.Map; @RequestMapping({"/api/chat/agent", "/openapi/chat/agent"}) public class AgentController { - @Autowired - private AgentService agentService; + @Autowired private AgentService agentService; @PostMapping - public Agent createAgent(@RequestBody Agent agent, - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse) { + public Agent createAgent( + @RequestBody Agent agent, + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); return agentService.createAgent(agent, user); } @PutMapping - public Agent updateAgent(@RequestBody Agent agent, - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse) { + public Agent updateAgent( + @RequestBody Agent agent, + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); return agentService.updateAgent(agent, user); } @@ -64,5 +66,4 @@ public class AgentController { public Map getToolTypes() { return AgentToolType.getToolTypes(); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java index b642283fb..4fe5714f6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.chat.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -20,41 +23,39 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; - @RestController @RequestMapping({"/api/chat/conf", "/openapi/chat/conf"}) public class ChatConfigController { - @Autowired - private ConfigService configService; + @Autowired private ConfigService configService; - @Autowired - private SemanticLayerService semanticLayerService; + @Autowired private SemanticLayerService semanticLayerService; @PostMapping - public Long addChatConfig(@RequestBody ChatConfigBaseReq extendBaseCmd, - HttpServletRequest request, - HttpServletResponse response) { + public Long addChatConfig( + @RequestBody ChatConfigBaseReq extendBaseCmd, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return configService.addConfig(extendBaseCmd, user); } @PutMapping - public Long editModelExtend(@RequestBody ChatConfigEditReqReq extendEditCmd, - HttpServletRequest request, - HttpServletResponse response) { + public Long editModelExtend( + @RequestBody ChatConfigEditReqReq extendEditCmd, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return configService.editConfig(extendEditCmd, user); } @PostMapping("/search") - public List search(@RequestBody ChatConfigFilter filter, - HttpServletRequest request, - HttpServletResponse response) { + public List search( + @RequestBody ChatConfigFilter filter, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return configService.search(filter, user); } @@ -78,5 +79,4 @@ public class ChatConfigController { public DataSetSchema getDataSetSchema(@PathVariable("id") Long id) { return semanticLayerService.getDataSetSchema(id); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java index 0ca44d025..d7794cd3f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.chat.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -17,65 +19,72 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @RequestMapping({"/api/chat/manage", "/openapi/chat/manage"}) public class ChatController { - @Autowired - private ChatManageService chatService; + @Autowired private ChatManageService chatService; @PostMapping("/save") - public Boolean save(@RequestParam(value = "chatName") String chatName, - @RequestParam(value = "agentId", required = false) Integer agentId, - HttpServletRequest request, HttpServletResponse response) { + public Boolean save( + @RequestParam(value = "chatName") String chatName, + @RequestParam(value = "agentId", required = false) Integer agentId, + HttpServletRequest request, + HttpServletResponse response) { chatService.addChat(UserHolder.findUser(request, response), chatName, agentId); return true; } @GetMapping("/getAll") - public List getAllConversions(@RequestParam(value = "agentId", required = false) Integer agentId, - HttpServletRequest request, HttpServletResponse response) { + public List getAllConversions( + @RequestParam(value = "agentId", required = false) Integer agentId, + HttpServletRequest request, + HttpServletResponse response) { String userName = UserHolder.findUser(request, response).getName(); return chatService.getAll(userName, agentId); } @PostMapping("/delete") - public Boolean deleteConversion(@RequestParam(value = "chatId") long chatId, - HttpServletRequest request, HttpServletResponse response) { + public Boolean deleteConversion( + @RequestParam(value = "chatId") long chatId, + HttpServletRequest request, + HttpServletResponse response) { String userName = UserHolder.findUser(request, response).getName(); return chatService.deleteChat(chatId, userName); } @PostMapping("/updateChatName") - public Boolean updateConversionName(@RequestParam(value = "chatId") Long chatId, + public Boolean updateConversionName( + @RequestParam(value = "chatId") Long chatId, @RequestParam(value = "chatName") String chatName, - HttpServletRequest request, HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { String userName = UserHolder.findUser(request, response).getName(); return chatService.updateChatName(chatId, chatName, userName); } @PostMapping("/updateQAFeedback") - public Boolean updateQAFeedback(@RequestParam(value = "id") Integer id, + public Boolean updateQAFeedback( + @RequestParam(value = "id") Integer id, @RequestParam(value = "score") Integer score, @RequestParam(value = "feedback", required = false) String feedback) { return chatService.updateFeedback(id, score, feedback); } @PostMapping("/updateChatIsTop") - public Boolean updateConversionIsTop(@RequestParam(value = "chatId") Long chatId, - @RequestParam(value = "isTop") int isTop) { + public Boolean updateConversionIsTop( + @RequestParam(value = "chatId") Long chatId, @RequestParam(value = "isTop") int isTop) { return chatService.updateChatIsTop(chatId, isTop); } @PostMapping("/pageQueryInfo") - public PageInfo pageQueryInfo(@RequestBody PageQueryInfoReq pageQueryInfoCommand, - @RequestParam(value = "chatId") long chatId, - HttpServletRequest request, - HttpServletResponse response) { + public PageInfo pageQueryInfo( + @RequestBody PageQueryInfoReq pageQueryInfoCommand, + @RequestParam(value = "chatId") long chatId, + HttpServletRequest request, + HttpServletResponse response) { pageQueryInfoCommand.setUserName(UserHolder.findUser(request, response).getName()); return chatService.queryInfo(pageQueryInfoCommand, chatId); } @@ -86,9 +95,9 @@ public class ChatController { } @PostMapping("/queryShowCase") - public ShowCaseResp queryShowCase(@RequestBody PageQueryInfoReq pageQueryInfoCommand, - @RequestParam(value = "agentId") int agentId) { + public ShowCaseResp queryShowCase( + @RequestBody PageQueryInfoReq pageQueryInfoCommand, + @RequestParam(value = "agentId") int agentId) { return chatService.queryShowCase(pageQueryInfoCommand, agentId); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java index 3e67ebc2b..e38ef5940 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.chat.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -19,45 +22,47 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; - -/** - * query controller - */ +/** query controller */ @RestController @RequestMapping({"/api/chat/query", "/openapi/chat/query"}) public class ChatQueryController { - @Autowired - private ChatQueryService chatQueryService; + @Autowired private ChatQueryService chatQueryService; @PostMapping("search") - public Object search(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request, - HttpServletResponse response) { + public Object search( + @RequestBody ChatParseReq chatParseReq, + HttpServletRequest request, + HttpServletResponse response) { chatParseReq.setUser(UserHolder.findUser(request, response)); return chatQueryService.search(chatParseReq); } @PostMapping("parse") - public Object parse(@RequestBody ChatParseReq chatParseReq, - HttpServletRequest request, HttpServletResponse response) throws Exception { + public Object parse( + @RequestBody ChatParseReq chatParseReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { chatParseReq.setUser(UserHolder.findUser(request, response)); return chatQueryService.performParsing(chatParseReq); } @PostMapping("execute") - public Object execute(@RequestBody ChatExecuteReq chatExecuteReq, - HttpServletRequest request, HttpServletResponse response) + public Object execute( + @RequestBody ChatExecuteReq chatExecuteReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { chatExecuteReq.setUser(UserHolder.findUser(request, response)); return chatQueryService.performExecution(chatExecuteReq); } @PostMapping("/") - public Object query(@RequestBody ChatParseReq chatParseReq, - HttpServletRequest request, HttpServletResponse response) + public Object query( + @RequestBody ChatParseReq chatParseReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); chatParseReq.setUser(user); @@ -75,17 +80,22 @@ public class ChatQueryController { } @PostMapping("queryData") - public Object queryData(@RequestBody ChatQueryDataReq chatQueryDataReq, - HttpServletRequest request, HttpServletResponse response) throws Exception { + public Object queryData( + @RequestBody ChatQueryDataReq chatQueryDataReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { chatQueryDataReq.setUser(UserHolder.findUser(request, response)); return chatQueryService.queryData(chatQueryDataReq, UserHolder.findUser(request, response)); } @PostMapping("queryDimensionValue") - public Object queryDimensionValue(@RequestBody @Valid DimensionValueReq dimensionValueReq, - HttpServletRequest request, HttpServletResponse response) throws Exception { - return chatQueryService.queryDimensionValue(dimensionValueReq, UserHolder.findUser(request, response)); + public Object queryDimensionValue( + @RequestBody @Valid DimensionValueReq dimensionValueReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { + return chatQueryService.queryDimensionValue( + dimensionValueReq, UserHolder.findUser(request, response)); } - } - diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java index 58f02a757..c0c60813d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.chat.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -13,20 +16,17 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - @RestController @RequestMapping({"/api/chat/memory"}) public class MemoryController { - @Autowired - private MemoryService memoryService; + @Autowired private MemoryService memoryService; @PostMapping("/updateMemory") - public Boolean updateMemory(@RequestBody ChatMemoryUpdateReq chatMemoryUpdateReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean updateMemory( + @RequestBody ChatMemoryUpdateReq chatMemoryUpdateReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); memoryService.updateMemory(chatMemoryUpdateReq, user); return true; @@ -36,5 +36,4 @@ public class MemoryController { public PageInfo pageMemories(@RequestBody PageMemoryReq pageMemoryReq) { return memoryService.pageMemories(pageMemoryReq); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java index 96717c2e5..9f4bf6390 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.chat.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -16,30 +19,29 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @RequestMapping("/api/chat/plugin") public class PluginController { - @Autowired - protected PluginService pluginService; + @Autowired protected PluginService pluginService; @PostMapping - public boolean createPlugin(@RequestBody ChatPlugin plugin, - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse) { + public boolean createPlugin( + @RequestBody ChatPlugin plugin, + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); pluginService.createPlugin(plugin, user); return true; } @PutMapping - public boolean updatePlugin(@RequestBody ChatPlugin plugin, - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse) { + public boolean updatePlugin( + @RequestBody ChatPlugin plugin, + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); pluginService.updatePlugin(plugin, user); return true; @@ -57,18 +59,18 @@ public class PluginController { } @PostMapping("/query") - List query(@RequestBody PluginQueryReq pluginQueryReq, - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse) { + List query( + @RequestBody PluginQueryReq pluginQueryReq, + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); return pluginService.queryWithAuthCheck(pluginQueryReq, user); } @AuthenticationIgnore @PostMapping("/pluginDemo") - public String pluginDemo(@RequestParam("queryText") String queryText, - @RequestBody Object object) { + public String pluginDemo( + @RequestParam("queryText") String queryText, @RequestBody Object object) { return String.format("已收到您的问题:%s, 但这只是一个demo~", queryText); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java index c44a87dee..98e32cbaa 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.server.agent.Agent; + import java.util.List; public interface AgentService { @@ -15,5 +16,4 @@ public interface AgentService { Agent getAgent(Integer id); void deleteAgent(Integer id); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatContextService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatContextService.java index 8ea2df1f7..1c4ef8cb0 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatContextService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatContextService.java @@ -7,5 +7,4 @@ public interface ChatContextService { ChatContext getOrCreateContext(Integer chatId); void updateContext(ChatContext chatCtx); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatManageService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatManageService.java index 28de00f4e..d01862ba9 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatManageService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatManageService.java @@ -6,13 +6,13 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import java.util.List; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatQueryService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatQueryService.java index 2ea2f70ec..07b29d467 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatQueryService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatQueryService.java @@ -4,9 +4,9 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SearchResult; import java.util.List; @@ -24,5 +24,4 @@ public interface ChatQueryService { Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception; Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ConfigService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ConfigService.java index 21bb6ba91..deefca664 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ConfigService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ConfigService.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.service; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; @@ -27,5 +26,4 @@ public interface ConfigService { ChatConfigResp fetchConfigByModelId(Long modelId); List getAllChatRichConfig(); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java index 9aa803404..3b3efc115 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.service; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; import com.tencent.supersonic.chat.server.plugin.ChatPlugin; @@ -28,5 +27,4 @@ public interface PluginService { List queryWithAuthCheck(PluginQueryReq pluginQueryReq, User user); Map getNameToPlugin(); - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/RecommendService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/RecommendService.java index 4ff49e3fe..6e4922049 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/RecommendService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/RecommendService.java @@ -3,11 +3,10 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.chat.api.pojo.request.RecommendReq; import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp; import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; + import java.util.List; -/*** - * Recommend Service - */ +/** * Recommend Service */ public interface RecommendService { RecommendResp recommend(RecommendReq recommendReq, Long limit); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index f7523d786..566d8d7b6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -29,21 +29,17 @@ import java.util.stream.Collectors; @Slf4j @Service -public class AgentServiceImpl extends ServiceImpl - implements AgentService { +public class AgentServiceImpl extends ServiceImpl implements AgentService { - @Autowired - private MemoryService memoryService; + @Autowired private MemoryService memoryService; - @Autowired - private ChatQueryService chatQueryService; + @Autowired private ChatQueryService chatQueryService; private ExecutorService executorService = Executors.newFixedThreadPool(1); @Override public List getAgents() { - return getAgentDOList().stream() - .map(this::convert).collect(Collectors.toList()); + return getAgentDOList().stream().map(this::convert).collect(Collectors.toList()); } @Override @@ -78,8 +74,8 @@ public class AgentServiceImpl extends ServiceImpl } /** - * the example in the agent will be executed by default, - * if the result is correct, it will be put into memory as a reference for LLM + * the example in the agent will be executed by default, if the result is correct, it will be + * put into memory as a reference for LLM * * @param agent */ @@ -88,16 +84,19 @@ public class AgentServiceImpl extends ServiceImpl } private synchronized void doExecuteAgentExamples(Agent agent) { - if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getModelConfig()) + if (!agent.containsLLMParserTool() + || !LLMConnHelper.testConnection(agent.getModelConfig()) || CollectionUtils.isEmpty(agent.getExamples())) { return; } List examples = agent.getExamples(); - ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId()) - .questions(examples).build(); - List memoriesExisted = memoryService.getMemories(chatMemoryFilter) - .stream().map(ChatMemoryDO::getQuestion).collect(Collectors.toList()); + ChatMemoryFilter chatMemoryFilter = + ChatMemoryFilter.builder().agentId(agent.getId()).questions(examples).build(); + List memoriesExisted = + memoryService.getMemories(chatMemoryFilter).stream() + .map(ChatMemoryDO::getQuestion) + .collect(Collectors.toList()); for (String example : examples) { if (memoriesExisted.contains(example)) { continue; @@ -124,7 +123,8 @@ public class AgentServiceImpl extends ServiceImpl agent.setExamples(JsonUtil.toList(agentDO.getExamples(), String.class)); agent.setModelConfig(JsonUtil.toObject(agentDO.getModelConfig(), ChatModelConfig.class)); agent.setPromptConfig(JsonUtil.toObject(agentDO.getPromptConfig(), PromptConfig.class)); - agent.setMultiTurnConfig(JsonUtil.toObject(agentDO.getMultiTurnConfig(), MultiTurnConfig.class)); + agent.setMultiTurnConfig( + JsonUtil.toObject(agentDO.getMultiTurnConfig(), MultiTurnConfig.class)); agent.setVisualConfig(JsonUtil.toObject(agentDO.getVisualConfig(), VisualConfig.class)); return agent; } @@ -143,5 +143,4 @@ public class AgentServiceImpl extends ServiceImpl } return agentDO; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatContextServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatContextServiceImpl.java index 194990eb5..52654cb9a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatContextServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatContextServiceImpl.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.service.impl; -import com.tencent.supersonic.chat.server.service.ChatContextService; -import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.chat.server.persistence.repository.ChatContextRepository; +import com.tencent.supersonic.chat.server.pojo.ChatContext; +import com.tencent.supersonic.chat.server.service.ChatContextService; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -26,5 +26,4 @@ public class ChatContextServiceImpl implements ChatContextService { log.debug("save ChatContext {}", chatCtx); chatContextRepository.updateContext(chatCtx); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatManageServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatManageServiceImpl.java index d9e24514f..77102cd10 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatManageServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatManageServiceImpl.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; @@ -18,7 +19,6 @@ import com.tencent.supersonic.chat.server.service.ChatManageService; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -33,15 +33,12 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; - @Slf4j @Service public class ChatManageServiceImpl implements ChatManageService { - @Autowired - private ChatRepository chatRepository; - @Autowired - private ChatQueryRepository chatQueryRepository; + @Autowired private ChatRepository chatRepository; + @Autowired private ChatQueryRepository chatQueryRepository; @Override public Long addChat(User user, String chatName, Integer agentId) { @@ -88,7 +85,8 @@ public class ChatManageServiceImpl implements ChatManageService { @Override public PageInfo queryInfo(PageQueryInfoReq pageQueryInfoReq, long chatId) { - PageInfo queryRespPageInfo = chatQueryRepository.getChatQuery(pageQueryInfoReq, chatId); + PageInfo queryRespPageInfo = + chatQueryRepository.getChatQuery(pageQueryInfoReq, chatId); if (CollectionUtils.isEmpty(queryRespPageInfo.getList())) { return queryRespPageInfo; } @@ -123,47 +121,62 @@ public class ChatManageServiceImpl implements ChatManageService { if (CollectionUtils.isEmpty(queryResps)) { return showCaseResp; } - queryResps.removeIf(queryResp -> { - if (queryResp.getQueryResult() == null) { - return true; - } - if (queryResp.getQueryResult().getResponse() != null) { - return false; - } - if (CollectionUtils.isEmpty(queryResp.getQueryResult().getQueryResults())) { - return true; - } - Map data = queryResp.getQueryResult().getQueryResults().get(0); - return CollectionUtils.isEmpty(data); - }); - queryResps = new ArrayList<>(queryResps.stream() - .collect(Collectors.toMap(QueryResp::getQueryText, Function.identity(), - (existing, replacement) -> existing, LinkedHashMap::new)).values()); + queryResps.removeIf( + queryResp -> { + if (queryResp.getQueryResult() == null) { + return true; + } + if (queryResp.getQueryResult().getResponse() != null) { + return false; + } + if (CollectionUtils.isEmpty(queryResp.getQueryResult().getQueryResults())) { + return true; + } + Map data = queryResp.getQueryResult().getQueryResults().get(0); + return CollectionUtils.isEmpty(data); + }); + queryResps = + new ArrayList<>( + queryResps.stream() + .collect( + Collectors.toMap( + QueryResp::getQueryText, + Function.identity(), + (existing, replacement) -> existing, + LinkedHashMap::new)) + .values()); fillParseInfo(queryResps); - Map> showCaseMap = queryResps.stream() - .collect(Collectors.groupingBy(QueryResp::getChatId)); + Map> showCaseMap = + queryResps.stream().collect(Collectors.groupingBy(QueryResp::getChatId)); showCaseResp.setShowCaseMap(showCaseMap); return showCaseResp; } private void fillParseInfo(List queryResps) { - List queryIds = queryResps.stream() - .map(QueryResp::getQuestionId).collect(Collectors.toList()); + List queryIds = + queryResps.stream().map(QueryResp::getQuestionId).collect(Collectors.toList()); List chatParseDOs = chatQueryRepository.getParseInfoList(queryIds); if (CollectionUtils.isEmpty(chatParseDOs)) { return; } - Map> chatParseMap = chatParseDOs.stream() - .collect(Collectors.groupingBy(ChatParseDO::getQuestionId)); + Map> chatParseMap = + chatParseDOs.stream().collect(Collectors.groupingBy(ChatParseDO::getQuestionId)); for (QueryResp queryResp : queryResps) { List chatParseDOList = chatParseMap.get(queryResp.getQuestionId()); if (CollectionUtils.isEmpty(chatParseDOList)) { continue; } - List parseInfos = chatParseDOList.stream().map(chatParseDO -> - JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class)) - .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed()) - .collect(Collectors.toList()); + List parseInfos = + chatParseDOList.stream() + .map( + chatParseDO -> + JsonUtil.toObject( + chatParseDO.getParseInfo(), + SemanticParseInfo.class)) + .sorted( + Comparator.comparingDouble(SemanticParseInfo::getScore) + .reversed()) + .collect(Collectors.toList()); queryResp.setParseInfos(parseInfos); } } @@ -175,8 +188,10 @@ public class ChatManageServiceImpl implements ChatManageService { chatQueryDO.setQueryResult(JsonUtil.toString(queryResult)); chatQueryDO.setQueryState(1); updateQuery(chatQueryDO); - chatRepository.updateLastQuestion(chatExecuteReq.getChatId().longValue(), - chatExecuteReq.getQueryText(), getCurrentTime()); + chatRepository.updateLastQuestion( + chatExecuteReq.getChatId().longValue(), + chatExecuteReq.getQueryText(), + getCurrentTime()); return chatQueryDO; } @@ -208,5 +223,4 @@ public class ChatManageServiceImpl implements ChatManageService { ChatParseDO chatParseDO = chatQueryRepository.getParseInfo(questionId, parseId); return JSONObject.parseObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java index 656233a8b..ec76547a7 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java @@ -75,24 +75,21 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; - @Slf4j @Service public class ChatQueryServiceImpl implements ChatQueryService { - @Autowired - private ChatManageService chatManageService; - @Autowired - private ChatLayerService chatLayerService; - @Autowired - private SemanticLayerService semanticLayerService; - @Autowired - private AgentService agentService; + @Autowired private ChatManageService chatManageService; + @Autowired private ChatLayerService chatLayerService; + @Autowired private SemanticLayerService semanticLayerService; + @Autowired private AgentService agentService; private List chatQueryParsers = ComponentFactory.getChatParsers(); private List chatQueryExecutors = ComponentFactory.getChatExecutors(); - private List parseResultProcessors = ComponentFactory.getParseProcessors(); - private List executeResultProcessors = ComponentFactory.getExecuteProcessors(); + private List parseResultProcessors = + ComponentFactory.getParseProcessors(); + private List executeResultProcessors = + ComponentFactory.getExecuteProcessors(); @Override public List search(ChatParseReq chatParseReq) { @@ -153,8 +150,11 @@ public class ChatQueryServiceImpl implements ChatQueryService { chatParseReq.setUser(User.getFakeUser()); ParseResp parseResp = performParsing(chatParseReq); if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) { - log.debug("chatId:{}, agentId:{}, queryText:{}, parseResp.getSelectedParses() is empty", - chatId, agentId, queryText); + log.debug( + "chatId:{}, agentId:{}, queryText:{}, parseResp.getSelectedParses() is empty", + chatId, + agentId, + queryText); return null; } ChatExecuteReq executeReq = new ChatExecuteReq(); @@ -185,8 +185,9 @@ public class ChatQueryServiceImpl implements ChatQueryService { private ExecuteContext buildExecuteContext(ChatExecuteReq chatExecuteReq) { ExecuteContext executeContext = new ExecuteContext(); BeanMapper.mapper(chatExecuteReq, executeContext); - SemanticParseInfo parseInfo = chatManageService.getParseInfo( - chatExecuteReq.getQueryId(), chatExecuteReq.getParseId()); + SemanticParseInfo parseInfo = + chatManageService.getParseInfo( + chatExecuteReq.getQueryId(), chatExecuteReq.getParseId()); Agent agent = agentService.getAgent(chatExecuteReq.getAgentId()); executeContext.setAgent(agent); executeContext.setParseInfo(parseInfo); @@ -196,9 +197,11 @@ public class ChatQueryServiceImpl implements ChatQueryService { @Override public Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception { Integer parseId = chatQueryDataReq.getParseId(); - SemanticParseInfo parseInfo = chatManageService.getParseInfo(chatQueryDataReq.getQueryId(), parseId); + SemanticParseInfo parseInfo = + chatManageService.getParseInfo(chatQueryDataReq.getQueryId(), parseId); parseInfo = mergeParseInfo(parseInfo, chatQueryDataReq); - DataSetSchema dataSetSchema = semanticLayerService.getDataSetSchema(parseInfo.getDataSetId()); + DataSetSchema dataSetSchema = + semanticLayerService.getDataSetSchema(parseInfo.getDataSetId()); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); semanticQuery.setParseInfo(parseInfo); @@ -220,9 +223,9 @@ public class ChatQueryServiceImpl implements ChatQueryService { return SqlSelectHelper.getAllSelectFields(sqlInfo.getCorrectedS2SQL()); } - private void handleLLMQueryMode(ChatQueryDataReq chatQueryDataReq, - SemanticQuery semanticQuery, - User user) throws Exception { + private void handleLLMQueryMode( + ChatQueryDataReq chatQueryDataReq, SemanticQuery semanticQuery, User user) + throws Exception { SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); List fields = getFieldsFromSql(parseInfo); if (checkMetricReplace(fields, chatQueryDataReq.getMetrics())) { @@ -240,18 +243,16 @@ public class ChatQueryServiceImpl implements ChatQueryService { } } - private void handleRuleQueryMode(SemanticQuery semanticQuery, - DataSetSchema dataSetSchema, - User user) { + private void handleRuleQueryMode( + SemanticQuery semanticQuery, DataSetSchema dataSetSchema, User user) { log.info("rule begin replace metrics and revise filters!"); validFilter(semanticQuery.getParseInfo().getDimensionFilters()); validFilter(semanticQuery.getParseInfo().getMetricFilters()); semanticQuery.initS2Sql(dataSetSchema, user); } - private QueryResult executeQuery(SemanticQuery semanticQuery, - User user, - DataSetSchema dataSetSchema) throws Exception { + private QueryResult executeQuery( + SemanticQuery semanticQuery, User user, DataSetSchema dataSetSchema) throws Exception { SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); QueryResult queryResult = doExecution(semanticQueryReq, parseInfo.getQueryMode(), user); @@ -266,7 +267,8 @@ public class ChatQueryServiceImpl implements ChatQueryService { if (CollectionUtils.isEmpty(oriFields) || CollectionUtils.isEmpty(metrics)) { return false; } - List metricNames = metrics.stream().map(SchemaElement::getName).collect(Collectors.toList()); + List metricNames = + metrics.stream().map(SchemaElement::getName).collect(Collectors.toList()); return !oriFields.containsAll(metricNames); } @@ -274,26 +276,41 @@ public class ChatQueryServiceImpl implements ChatQueryService { String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL(); log.info("correctorSql before replacing:{}", correctorSql); // get where filter and having filter - List whereExpressionList = SqlSelectHelper.getWhereExpressions(correctorSql); + List whereExpressionList = + SqlSelectHelper.getWhereExpressions(correctorSql); // replace where filter List addWhereConditions = new ArrayList<>(); - Set removeWhereFieldNames = updateFilters(whereExpressionList, queryData.getDimensionFilters(), - parseInfo.getDimensionFilters(), addWhereConditions); + Set removeWhereFieldNames = + updateFilters( + whereExpressionList, + queryData.getDimensionFilters(), + parseInfo.getDimensionFilters(), + addWhereConditions); Map> filedNameToValueMap = new HashMap<>(); - Set removeDataFieldNames = updateDateInfo(queryData, parseInfo, filedNameToValueMap, - whereExpressionList, addWhereConditions); + Set removeDataFieldNames = + updateDateInfo( + queryData, + parseInfo, + filedNameToValueMap, + whereExpressionList, + addWhereConditions); removeWhereFieldNames.addAll(removeDataFieldNames); correctorSql = SqlReplaceHelper.replaceValue(correctorSql, filedNameToValueMap); correctorSql = SqlRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames); // replace having filter - List havingExpressionList = SqlSelectHelper.getHavingExpressions(correctorSql); + List havingExpressionList = + SqlSelectHelper.getHavingExpressions(correctorSql); List addHavingConditions = new ArrayList<>(); - Set removeHavingFieldNames = updateFilters(havingExpressionList, - queryData.getDimensionFilters(), parseInfo.getDimensionFilters(), addHavingConditions); + Set removeHavingFieldNames = + updateFilters( + havingExpressionList, + queryData.getDimensionFilters(), + parseInfo.getDimensionFilters(), + addHavingConditions); correctorSql = SqlReplaceHelper.replaceHavingValue(correctorSql, new HashMap<>()); correctorSql = SqlRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames); @@ -304,8 +321,10 @@ public class ChatQueryServiceImpl implements ChatQueryService { } private void replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) { - List oriMetrics = parseInfo.getMetrics().stream() - .map(SchemaElement::getName).collect(Collectors.toList()); + List oriMetrics = + parseInfo.getMetrics().stream() + .map(SchemaElement::getName) + .collect(Collectors.toList()); String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL(); log.info("before replaceMetrics:{}", correctorSql); log.info("filteredMetrics:{},metrics:{}", oriMetrics, metric); @@ -318,7 +337,8 @@ public class ChatQueryServiceImpl implements ChatQueryService { parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql); } - private QueryResult doExecution(SemanticQueryReq semanticQueryReq, String queryMode, User user) throws Exception { + private QueryResult doExecution(SemanticQueryReq semanticQueryReq, String queryMode, User user) + throws Exception { SemanticQueryResp queryResp = semanticLayerService.queryByReq(semanticQueryReq, user); QueryResult queryResult = new QueryResult(); @@ -337,16 +357,20 @@ public class ChatQueryServiceImpl implements ChatQueryService { return queryResult; } - private Set updateDateInfo(ChatQueryDataReq queryData, SemanticParseInfo parseInfo, - Map> filedNameToValueMap, - List fieldExpressionList, - List addConditions) { + private Set updateDateInfo( + ChatQueryDataReq queryData, + SemanticParseInfo parseInfo, + Map> filedNameToValueMap, + List fieldExpressionList, + List addConditions) { Set removeFieldNames = new HashSet<>(); if (Objects.isNull(queryData.getDateInfo())) { return removeFieldNames; } if (queryData.getDateInfo().getUnit() > 1) { - queryData.getDateInfo().setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1)); + queryData + .getDateInfo() + .setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1)); queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(1)); } // startDate equals to endDate @@ -355,17 +379,20 @@ public class ChatQueryServiceImpl implements ChatQueryService { // first remove,then add removeFieldNames.add(TimeDimensionEnum.DAY.getChName()); GreaterThanEquals greaterThanEquals = new GreaterThanEquals(); - addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions); + addTimeFilters( + queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions); MinorThanEquals minorThanEquals = new MinorThanEquals(); - addTimeFilters(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions); + addTimeFilters( + queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions); break; } } for (FieldExpression fieldExpression : fieldExpressionList) { for (QueryFilter queryFilter : queryData.getDimensionFilters()) { if (queryFilter.getOperator().equals(FilterOperatorEnum.LIKE) - && FilterOperatorEnum.LIKE.getValue().equalsIgnoreCase( - fieldExpression.getOperator())) { + && FilterOperatorEnum.LIKE + .getValue() + .equalsIgnoreCase(fieldExpression.getOperator())) { Map replaceMap = new HashMap<>(); String preValue = fieldExpression.getFieldValue().toString(); String curValue = queryFilter.getValue().toString(); @@ -385,9 +412,8 @@ public class ChatQueryServiceImpl implements ChatQueryService { return removeFieldNames; } - private void addTimeFilters(String date, - T comparisonExpression, - List addConditions) { + private void addTimeFilters( + String date, T comparisonExpression, List addConditions) { Column column = new Column(TimeDimensionEnum.DAY.getChName()); StringValue stringValue = new StringValue(date); comparisonExpression.setLeftExpression(column); @@ -395,10 +421,11 @@ public class ChatQueryServiceImpl implements ChatQueryService { addConditions.add(comparisonExpression); } - private Set updateFilters(List fieldExpressionList, - Set metricFilters, - Set contextMetricFilters, - List addConditions) { + private Set updateFilters( + List fieldExpressionList, + Set metricFilters, + Set contextMetricFilters, + List addConditions) { Set removeFieldNames = new HashSet<>(); if (CollectionUtils.isEmpty(metricFilters)) { return removeFieldNames; @@ -417,13 +444,15 @@ public class ChatQueryServiceImpl implements ChatQueryService { return removeFieldNames; } - private void handleFilter(QueryFilter dslQueryFilter, - Set contextMetricFilters, - List addConditions) { + private void handleFilter( + QueryFilter dslQueryFilter, + Set contextMetricFilters, + List addConditions) { FilterOperatorEnum operator = dslQueryFilter.getOperator(); if (operator == FilterOperatorEnum.IN) { - addWhereInFilters(dslQueryFilter, new InExpression(), contextMetricFilters, addConditions); + addWhereInFilters( + dslQueryFilter, new InExpression(), contextMetricFilters, addConditions); } else { ComparisonOperator expression = FilterOperatorEnum.createExpression(operator); if (Objects.nonNull(expression)) { @@ -433,37 +462,43 @@ public class ChatQueryServiceImpl implements ChatQueryService { } // add in condition to sql where condition - private void addWhereInFilters(QueryFilter dslQueryFilter, - InExpression inExpression, - Set contextMetricFilters, - List addConditions) { + private void addWhereInFilters( + QueryFilter dslQueryFilter, + InExpression inExpression, + Set contextMetricFilters, + List addConditions) { Column column = new Column(dslQueryFilter.getName()); ParenthesedExpressionList parenthesedExpressionList = new ParenthesedExpressionList<>(); - List valueList = JsonUtil.toList( - JsonUtil.toString(dslQueryFilter.getValue()), String.class); + List valueList = + JsonUtil.toList(JsonUtil.toString(dslQueryFilter.getValue()), String.class); if (CollectionUtils.isEmpty(valueList)) { return; } - valueList.stream().forEach(o -> { - StringValue stringValue = new StringValue(o); - parenthesedExpressionList.add(stringValue); - }); + valueList.stream() + .forEach( + o -> { + StringValue stringValue = new StringValue(o); + parenthesedExpressionList.add(stringValue); + }); inExpression.setLeftExpression(column); inExpression.setRightExpression(parenthesedExpressionList); addConditions.add(inExpression); - contextMetricFilters.stream().forEach(o -> { - if (o.getName().equals(dslQueryFilter.getName())) { - o.setValue(dslQueryFilter.getValue()); - o.setOperator(dslQueryFilter.getOperator()); - } - }); + contextMetricFilters.stream() + .forEach( + o -> { + if (o.getName().equals(dslQueryFilter.getName())) { + o.setValue(dslQueryFilter.getValue()); + o.setOperator(dslQueryFilter.getOperator()); + } + }); } // add where filter - private void addWhereFilters(QueryFilter dslQueryFilter, - ComparisonOperator comparisonExpression, - Set contextMetricFilters, - List addConditions) { + private void addWhereFilters( + QueryFilter dslQueryFilter, + ComparisonOperator comparisonExpression, + Set contextMetricFilters, + List addConditions) { String columnName = dslQueryFilter.getName(); if (StringUtils.isNotBlank(dslQueryFilter.getFunction())) { columnName = dslQueryFilter.getFunction() + "(" + dslQueryFilter.getName() + ")"; @@ -474,23 +509,26 @@ public class ChatQueryServiceImpl implements ChatQueryService { Column column = new Column(columnName); comparisonExpression.setLeftExpression(column); if (StringUtils.isNumeric(dslQueryFilter.getValue().toString())) { - LongValue longValue = new LongValue(Long.parseLong(dslQueryFilter.getValue().toString())); + LongValue longValue = + new LongValue(Long.parseLong(dslQueryFilter.getValue().toString())); comparisonExpression.setRightExpression(longValue); } else { StringValue stringValue = new StringValue(dslQueryFilter.getValue().toString()); comparisonExpression.setRightExpression(stringValue); } addConditions.add(comparisonExpression); - contextMetricFilters.stream().forEach(o -> { - if (o.getName().equals(dslQueryFilter.getName())) { - o.setValue(dslQueryFilter.getValue()); - o.setOperator(dslQueryFilter.getOperator()); - } - }); + contextMetricFilters.stream() + .forEach( + o -> { + if (o.getName().equals(dslQueryFilter.getName())) { + o.setValue(dslQueryFilter.getValue()); + o.setOperator(dslQueryFilter.getOperator()); + } + }); } - private SemanticParseInfo mergeParseInfo(SemanticParseInfo parseInfo, - ChatQueryDataReq queryData) { + private SemanticParseInfo mergeParseInfo( + SemanticParseInfo parseInfo, ChatQueryDataReq queryData) { if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { return parseInfo; } @@ -521,7 +559,8 @@ public class ChatQueryServiceImpl implements ChatQueryService { iterator.remove(); continue; } - List collection = JsonUtil.toList(JsonUtil.toString(queryFilterValue), String.class); + List collection = + JsonUtil.toList(JsonUtil.toString(queryFilterValue), String.class); if (FilterOperatorEnum.IN.equals(queryFilter.getOperator()) && CollectionUtils.isEmpty(collection)) { iterator.remove(); @@ -538,11 +577,10 @@ public class ChatQueryServiceImpl implements ChatQueryService { } public void saveQueryResult(ChatExecuteReq chatExecuteReq, QueryResult queryResult) { - //The history record only retains the query result of the first parse + // The history record only retains the query result of the first parse if (chatExecuteReq.getParseId() > 1) { return; } chatManageService.saveQueryResult(chatExecuteReq, queryResult); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 46364aa4d..0fd2ed66d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.service.impl; - import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; @@ -26,11 +25,11 @@ import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.chat.server.util.ChatConfigHelper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -44,7 +43,6 @@ import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; - @Slf4j @Service public class ConfigServiceImpl implements ConfigService { @@ -53,9 +51,10 @@ public class ConfigServiceImpl implements ConfigService { private final ChatConfigHelper chatConfigHelper; private final SemanticLayerService semanticLayerService; - - public ConfigServiceImpl(ChatConfigRepository chatConfigRepository, - ChatConfigHelper chatConfigHelper, SemanticLayerService semanticLayerService) { + public ConfigServiceImpl( + ChatConfigRepository chatConfigRepository, + ChatConfigHelper chatConfigHelper, + SemanticLayerService semanticLayerService) { this.chatConfigRepository = chatConfigRepository; this.chatConfigHelper = chatConfigHelper; this.semanticLayerService = semanticLayerService; @@ -81,9 +80,11 @@ public class ConfigServiceImpl implements ConfigService { @Override public Long editConfig(ChatConfigEditReqReq configEditCmd, User user) { log.info("[edit model extend] object:{}", JsonUtil.toString(configEditCmd, true)); - if (Objects.isNull(configEditCmd) || Objects.isNull(configEditCmd.getId()) && Objects.isNull( - configEditCmd.getModelId())) { - throw new RuntimeException("editConfig, id and modelId are not allowed to be empty at the same time"); + if (Objects.isNull(configEditCmd) + || Objects.isNull(configEditCmd.getId()) + && Objects.isNull(configEditCmd.getModelId())) { + throw new RuntimeException( + "editConfig, id and modelId are not allowed to be empty at the same time"); } ChatConfig chaConfig = chatConfigHelper.editChatConfig(configEditCmd, user); chatConfigRepository.updateConfig(chaConfig); @@ -106,39 +107,51 @@ public class ConfigServiceImpl implements ConfigService { List blackDimIdList = new ArrayList<>(); if (Objects.nonNull(chatConfig.getChatAggConfig()) && Objects.nonNull(chatConfig.getChatAggConfig().getVisibility())) { - blackDimIdList.addAll(chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList()); + blackDimIdList.addAll( + chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList()); } if (Objects.nonNull(chatConfig.getChatDetailConfig()) && Objects.nonNull(chatConfig.getChatDetailConfig().getVisibility())) { - blackDimIdList.addAll(chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList()); + blackDimIdList.addAll( + chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList()); } - List filterDimIdList = blackDimIdList.stream().distinct().collect(Collectors.toList()); + List filterDimIdList = + blackDimIdList.stream().distinct().collect(Collectors.toList()); List blackMetricIdList = new ArrayList<>(); if (Objects.nonNull(chatConfig.getChatAggConfig()) && Objects.nonNull(chatConfig.getChatAggConfig().getVisibility())) { - blackMetricIdList.addAll(chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList()); + blackMetricIdList.addAll( + chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList()); } if (Objects.nonNull(chatConfig.getChatDetailConfig()) && Objects.nonNull(chatConfig.getChatDetailConfig().getVisibility())) { - blackMetricIdList.addAll(chatConfig.getChatDetailConfig().getVisibility().getBlackMetricIdList()); + blackMetricIdList.addAll( + chatConfig.getChatDetailConfig().getVisibility().getBlackMetricIdList()); } - List filterMetricIdList = blackMetricIdList.stream().distinct().collect(Collectors.toList()); + List filterMetricIdList = + blackMetricIdList.stream().distinct().collect(Collectors.toList()); ItemNameVisibilityInfo itemNameVisibility = new ItemNameVisibilityInfo(); MetaFilter metaFilter = new MetaFilter(); metaFilter.setModelIds(Lists.newArrayList(modelId)); if (!CollectionUtils.isEmpty(blackDimIdList)) { List dimensionRespList = semanticLayerService.getDimensions(metaFilter); - List blackDimNameList = dimensionRespList.stream().filter(o -> filterDimIdList.contains(o.getId())) - .map(SchemaItem::getName).collect(Collectors.toList()); + List blackDimNameList = + dimensionRespList.stream() + .filter(o -> filterDimIdList.contains(o.getId())) + .map(SchemaItem::getName) + .collect(Collectors.toList()); itemNameVisibility.setBlackDimNameList(blackDimNameList); } if (!CollectionUtils.isEmpty(blackMetricIdList)) { List metricRespList = semanticLayerService.getMetrics(metaFilter); - List blackMetricList = metricRespList.stream().filter(o -> filterMetricIdList.contains(o.getId())) - .map(SchemaItem::getName).collect(Collectors.toList()); + List blackMetricList = + metricRespList.stream() + .filter(o -> filterMetricIdList.contains(o.getId())) + .map(SchemaItem::getName) + .collect(Collectors.toList()); itemNameVisibility.setBlackMetricNameList(blackMetricList); } return itemNameVisibility; @@ -156,8 +169,8 @@ public class ConfigServiceImpl implements ConfigService { return chatConfigRepository.getConfigByModelId(modelId); } - private ItemVisibilityInfo fetchVisibilityDescByConfig(ItemVisibility visibility, - DataSetSchema modelSchema) { + private ItemVisibilityInfo fetchVisibilityDescByConfig( + ItemVisibility visibility, DataSetSchema modelSchema) { ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo(); List dimIdAllList = chatConfigHelper.generateAllDimIdList(modelSchema); @@ -173,16 +186,22 @@ public class ConfigServiceImpl implements ConfigService { blackMetricIdList.addAll(visibility.getBlackMetricIdList()); } } - List whiteMetricIdList = metricIdAllList.stream() - .filter(id -> !blackMetricIdList.contains(id) && metricIdAllList.contains(id)) - .collect(Collectors.toList()); - List whiteDimIdList = dimIdAllList.stream() - .filter(id -> !blackDimIdList.contains(id) && dimIdAllList.contains(id)) - .collect(Collectors.toList()); + List whiteMetricIdList = + metricIdAllList.stream() + .filter( + id -> + !blackMetricIdList.contains(id) + && metricIdAllList.contains(id)) + .collect(Collectors.toList()); + List whiteDimIdList = + dimIdAllList.stream() + .filter(id -> !blackDimIdList.contains(id) && dimIdAllList.contains(id)) + .collect(Collectors.toList()); itemVisibilityDesc.setBlackDimIdList(blackDimIdList); itemVisibilityDesc.setBlackMetricIdList(blackMetricIdList); - itemVisibilityDesc.setWhiteDimIdList(Objects.isNull(whiteDimIdList) ? new ArrayList<>() : whiteDimIdList); + itemVisibilityDesc.setWhiteDimIdList( + Objects.isNull(whiteDimIdList) ? new ArrayList<>() : whiteDimIdList); itemVisibilityDesc.setWhiteMetricIdList( Objects.isNull(whiteMetricIdList) ? new ArrayList<>() : whiteMetricIdList); @@ -207,26 +226,31 @@ public class ConfigServiceImpl implements ConfigService { chatConfigRich.setModelName(dataSetSchema.getDataSet().getName()); chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(dataSetSchema, chatConfigResp)); - chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(dataSetSchema, chatConfigRich, chatConfigResp)); + chatConfigRich.setChatDetailRichConfig( + fillChatDetailRichConfig(dataSetSchema, chatConfigRich, chatConfigResp)); return chatConfigRich; } - private ChatDetailRichConfigResp fillChatDetailRichConfig(DataSetSchema modelSchema, - ChatConfigRichResp chatConfigRich, - ChatConfigResp chatConfigResp) { - if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) { + private ChatDetailRichConfigResp fillChatDetailRichConfig( + DataSetSchema modelSchema, + ChatConfigRichResp chatConfigRich, + ChatConfigResp chatConfigResp) { + if (Objects.isNull(chatConfigResp) + || Objects.isNull(chatConfigResp.getChatDetailConfig())) { return null; } ChatDetailRichConfigResp detailRichConfig = new ChatDetailRichConfigResp(); ChatDetailConfigReq chatDetailConfig = chatConfigResp.getChatDetailConfig(); - ItemVisibilityInfo itemVisibilityInfo = fetchVisibilityDescByConfig( - chatDetailConfig.getVisibility(), modelSchema); + ItemVisibilityInfo itemVisibilityInfo = + fetchVisibilityDescByConfig(chatDetailConfig.getVisibility(), modelSchema); detailRichConfig.setVisibility(itemVisibilityInfo); - detailRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatDetailConfig.getKnowledgeInfos(), modelSchema)); + detailRichConfig.setKnowledgeInfos( + fillKnowledgeBizName(chatDetailConfig.getKnowledgeInfos(), modelSchema)); detailRichConfig.setGlobalKnowledgeConfig(chatDetailConfig.getGlobalKnowledgeConfig()); - detailRichConfig.setChatDefaultConfig(fetchDefaultConfig(chatDetailConfig.getChatDefaultConfig(), - modelSchema, itemVisibilityInfo)); + detailRichConfig.setChatDefaultConfig( + fetchDefaultConfig( + chatDetailConfig.getChatDefaultConfig(), modelSchema, itemVisibilityInfo)); return detailRichConfig; } @@ -237,30 +261,38 @@ public class ConfigServiceImpl implements ConfigService { return entityRichInfo; } BeanUtils.copyProperties(entity, entityRichInfo); - Map dimIdAndRespPair = modelSchema.getDimensions().stream() - .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); + Map dimIdAndRespPair = + modelSchema.getDimensions().stream() + .collect( + Collectors.toMap( + SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId())); return entityRichInfo; } - private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema, ChatConfigResp chatConfigResp) { + private ChatAggRichConfigResp fillChatAggRichConfig( + DataSetSchema modelSchema, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) { return null; } ChatAggConfigReq chatAggConfig = chatConfigResp.getChatAggConfig(); ChatAggRichConfigResp chatAggRichConfig = new ChatAggRichConfigResp(); - ItemVisibilityInfo itemVisibilityInfo = fetchVisibilityDescByConfig(chatAggConfig.getVisibility(), modelSchema); + ItemVisibilityInfo itemVisibilityInfo = + fetchVisibilityDescByConfig(chatAggConfig.getVisibility(), modelSchema); chatAggRichConfig.setVisibility(itemVisibilityInfo); - chatAggRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatAggConfig.getKnowledgeInfos(), modelSchema)); + chatAggRichConfig.setKnowledgeInfos( + fillKnowledgeBizName(chatAggConfig.getKnowledgeInfos(), modelSchema)); chatAggRichConfig.setGlobalKnowledgeConfig(chatAggConfig.getGlobalKnowledgeConfig()); - chatAggRichConfig.setChatDefaultConfig(fetchDefaultConfig(chatAggConfig.getChatDefaultConfig(), - modelSchema, itemVisibilityInfo)); + chatAggRichConfig.setChatDefaultConfig( + fetchDefaultConfig( + chatAggConfig.getChatDefaultConfig(), modelSchema, itemVisibilityInfo)); return chatAggRichConfig; } - private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig, + private ChatDefaultRichConfigResp fetchDefaultConfig( + ChatDefaultConfigReq chatDefaultConfig, DataSetSchema modelSchema, ItemVisibilityInfo itemVisibilityInfo) { ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp(); @@ -268,41 +300,56 @@ public class ConfigServiceImpl implements ConfigService { return defaultRichConfig; } BeanUtils.copyProperties(chatDefaultConfig, defaultRichConfig); - Map dimIdAndRespPair = modelSchema.getDimensions().stream() - .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); + Map dimIdAndRespPair = + modelSchema.getDimensions().stream() + .collect( + Collectors.toMap( + SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); - Map metricIdAndRespPair = modelSchema.getMetrics().stream() - .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); + Map metricIdAndRespPair = + modelSchema.getMetrics().stream() + .collect( + Collectors.toMap( + SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); List dimensions = new ArrayList<>(); List metrics = new ArrayList<>(); if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensionIds())) { chatDefaultConfig.getDimensionIds().stream() - .filter(dimId -> dimIdAndRespPair.containsKey(dimId) - && itemVisibilityInfo.getWhiteDimIdList().contains(dimId)) - .forEach(dimId -> { - SchemaElement dimSchemaResp = dimIdAndRespPair.get(dimId); - if (Objects.nonNull(dimSchemaResp)) { - SchemaElement dimSchema = new SchemaElement(); - BeanUtils.copyProperties(dimSchemaResp, dimSchema); - dimensions.add(dimSchema); - } - - }); + .filter( + dimId -> + dimIdAndRespPair.containsKey(dimId) + && itemVisibilityInfo + .getWhiteDimIdList() + .contains(dimId)) + .forEach( + dimId -> { + SchemaElement dimSchemaResp = dimIdAndRespPair.get(dimId); + if (Objects.nonNull(dimSchemaResp)) { + SchemaElement dimSchema = new SchemaElement(); + BeanUtils.copyProperties(dimSchemaResp, dimSchema); + dimensions.add(dimSchema); + } + }); } if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetricIds())) { chatDefaultConfig.getMetricIds().stream() - .filter(metricId -> metricIdAndRespPair.containsKey(metricId) - && itemVisibilityInfo.getWhiteMetricIdList().contains(metricId)) - .forEach(metricId -> { - SchemaElement metricSchemaResp = metricIdAndRespPair.get(metricId); - if (Objects.nonNull(metricSchemaResp)) { - SchemaElement metricSchema = new SchemaElement(); - BeanUtils.copyProperties(metricSchemaResp, metricSchema); - metrics.add(metricSchema); - } - }); + .filter( + metricId -> + metricIdAndRespPair.containsKey(metricId) + && itemVisibilityInfo + .getWhiteMetricIdList() + .contains(metricId)) + .forEach( + metricId -> { + SchemaElement metricSchemaResp = metricIdAndRespPair.get(metricId); + if (Objects.nonNull(metricSchemaResp)) { + SchemaElement metricSchema = new SchemaElement(); + BeanUtils.copyProperties(metricSchemaResp, metricSchema); + metrics.add(metricSchema); + } + }); } defaultRichConfig.setDimensions(dimensions); @@ -310,21 +357,27 @@ public class ConfigServiceImpl implements ConfigService { return defaultRichConfig; } - private List fillKnowledgeBizName(List knowledgeInfos, - DataSetSchema modelSchema) { + private List fillKnowledgeBizName( + List knowledgeInfos, DataSetSchema modelSchema) { if (CollectionUtils.isEmpty(knowledgeInfos)) { return new ArrayList<>(); } - Map dimIdAndRespPair = modelSchema.getDimensions().stream() - .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); - knowledgeInfos.stream().forEach(knowledgeInfo -> { - if (Objects.nonNull(knowledgeInfo)) { - SchemaElement dimSchemaResp = dimIdAndRespPair.get(knowledgeInfo.getItemId()); - if (Objects.nonNull(dimSchemaResp)) { - knowledgeInfo.setBizName(dimSchemaResp.getBizName()); - } - } - }); + Map dimIdAndRespPair = + modelSchema.getDimensions().stream() + .collect( + Collectors.toMap( + SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); + knowledgeInfos.stream() + .forEach( + knowledgeInfo -> { + if (Objects.nonNull(knowledgeInfo)) { + SchemaElement dimSchemaResp = + dimIdAndRespPair.get(knowledgeInfo.getItemId()); + if (Objects.nonNull(dimSchemaResp)) { + knowledgeInfo.setBizName(dimSchemaResp.getBizName()); + } + } + }); return knowledgeInfos; } @@ -332,5 +385,4 @@ public class ConfigServiceImpl implements ConfigService { public List getAllChatRichConfig() { return new ArrayList<>(); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java index 182a7f4e4..41738e877 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java @@ -15,23 +15,21 @@ import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.pojo.Text2SQLExemplar; import com.tencent.supersonic.common.service.ExemplarService; import com.tencent.supersonic.common.util.BeanMapper; -import java.util.List; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.List; + @Service public class MemoryServiceImpl implements MemoryService { - @Autowired - private ChatMemoryRepository chatMemoryRepository; + @Autowired private ChatMemoryRepository chatMemoryRepository; - @Autowired - private ExemplarService exemplarService; + @Autowired private ExemplarService exemplarService; - @Autowired - private EmbeddingConfig embeddingConfig; + @Autowired private EmbeddingConfig embeddingConfig; @Override public void createMemory(ChatMemoryDO memory) { @@ -59,8 +57,7 @@ public class MemoryServiceImpl implements MemoryService { @Override public PageInfo pageMemories(PageMemoryReq pageMemoryReq) { - return PageHelper.startPage(pageMemoryReq.getCurrent(), - pageMemoryReq.getPageSize()) + return PageHelper.startPage(pageMemoryReq.getCurrent(), pageMemoryReq.getPageSize()) .doSelectPageInfo(() -> getMemories(pageMemoryReq.getChatMemoryFilter())); } @@ -80,10 +77,14 @@ public class MemoryServiceImpl implements MemoryService { queryWrapper.lambda().eq(ChatMemoryDO::getStatus, chatMemoryFilter.getStatus()); } if (chatMemoryFilter.getHumanReviewRet() != null) { - queryWrapper.lambda().eq(ChatMemoryDO::getHumanReviewRet, chatMemoryFilter.getHumanReviewRet()); + queryWrapper + .lambda() + .eq(ChatMemoryDO::getHumanReviewRet, chatMemoryFilter.getHumanReviewRet()); } if (chatMemoryFilter.getLlmReviewRet() != null) { - queryWrapper.lambda().eq(ChatMemoryDO::getLlmReviewRet, chatMemoryFilter.getLlmReviewRet()); + queryWrapper + .lambda() + .eq(ChatMemoryDO::getLlmReviewRet, chatMemoryFilter.getLlmReviewRet()); } return chatMemoryRepository.getMemories(queryWrapper); } @@ -91,7 +92,9 @@ public class MemoryServiceImpl implements MemoryService { @Override public List getMemoriesForLlmReview() { QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING) + queryWrapper + .lambda() + .eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING) .isNull(ChatMemoryDO::getLlmReviewRet); return chatMemoryRepository.getMemories(queryWrapper); } @@ -99,7 +102,8 @@ public class MemoryServiceImpl implements MemoryService { @Override public void enableMemory(ChatMemoryDO memory) { memory.setStatus(MemoryStatus.ENABLED); - exemplarService.storeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()), + exemplarService.storeExemplar( + embeddingConfig.getMemoryCollectionName(memory.getAgentId()), Text2SQLExemplar.builder() .question(memory.getQuestion()) .sideInfo(memory.getSideInfo()) @@ -111,7 +115,8 @@ public class MemoryServiceImpl implements MemoryService { @Override public void disableMemory(ChatMemoryDO memory) { memory.setStatus(MemoryStatus.DISABLED); - exemplarService.removeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()), + exemplarService.removeExemplar( + embeddingConfig.getMemoryCollectionName(memory.getAgentId()), Text2SQLExemplar.builder() .question(memory.getQuestion()) .sideInfo(memory.getSideInfo()) @@ -119,5 +124,4 @@ public class MemoryServiceImpl implements MemoryService { .sql(memory.getS2sql()) .build()); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java index def5b2e99..d642ec034 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java @@ -4,13 +4,13 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; +import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; +import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository; import com.tencent.supersonic.chat.server.plugin.ChatPlugin; import com.tencent.supersonic.chat.server.plugin.PluginParseConfig; import com.tencent.supersonic.chat.server.plugin.event.PluginAddEvent; import com.tencent.supersonic.chat.server.plugin.event.PluginDelEvent; import com.tencent.supersonic.chat.server.plugin.event.PluginUpdateEvent; -import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; -import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository; import com.tencent.supersonic.chat.server.service.PluginService; import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; @@ -36,8 +36,8 @@ public class PluginServiceImpl implements PluginService { private ApplicationEventPublisher publisher; - public PluginServiceImpl(PluginRepository pluginRepository, - ApplicationEventPublisher publisher) { + public PluginServiceImpl( + PluginRepository pluginRepository, ApplicationEventPublisher publisher) { this.pluginRepository = pluginRepository; this.publisher = publisher; } @@ -46,7 +46,7 @@ public class PluginServiceImpl implements PluginService { public synchronized void createPlugin(ChatPlugin plugin, User user) { PluginDO pluginDO = convert(plugin, user); pluginRepository.createPlugin(pluginDO); - //compatible with H2 db + // compatible with H2 db List plugins = getPluginList(); publisher.publishEvent(new PluginAddEvent(this, plugins.get(plugins.size() - 1))); } @@ -110,11 +110,18 @@ public class PluginServiceImpl implements PluginService { } List pluginDOS = pluginRepository.query(queryWrapper); if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) { - pluginDOS = pluginDOS.stream().filter(pluginDO -> - pluginDO.getPattern().contains(pluginQueryReq.getPattern()) - || (pluginDO.getName() != null - && pluginDO.getName().contains(pluginQueryReq.getPattern()))) - .collect(Collectors.toList()); + pluginDOS = + pluginDOS.stream() + .filter( + pluginDO -> + pluginDO.getPattern() + .contains(pluginQueryReq.getPattern()) + || (pluginDO.getName() != null + && pluginDO.getName() + .contains( + pluginQueryReq + .getPattern()))) + .collect(Collectors.toList()); } return convertList(pluginDOS); } @@ -123,13 +130,14 @@ public class PluginServiceImpl implements PluginService { public Optional getPluginByName(String name) { log.info("name:{}", name); return getPluginList().stream() - .filter(plugin -> { - PluginParseConfig functionCallConfig = getPluginParseConfig(plugin); - if (functionCallConfig == null) { - return false; - } - return functionCallConfig.getName().equalsIgnoreCase(name); - }) + .filter( + plugin -> { + PluginParseConfig functionCallConfig = getPluginParseConfig(plugin); + if (functionCallConfig == null) { + return false; + } + return functionCallConfig.getName().equalsIgnoreCase(name); + }) .findFirst(); } @@ -137,9 +145,10 @@ public class PluginServiceImpl implements PluginService { if (StringUtils.isBlank(plugin.getParseModeConfig())) { return null; } - PluginParseConfig functionCallConfig = JsonUtil.toObject( - plugin.getParseModeConfig(), PluginParseConfig.class); - if (Objects.isNull(functionCallConfig) || StringUtils.isEmpty(functionCallConfig.getName())) { + PluginParseConfig functionCallConfig = + JsonUtil.toObject(plugin.getParseModeConfig(), PluginParseConfig.class); + if (Objects.isNull(functionCallConfig) + || StringUtils.isEmpty(functionCallConfig.getName())) { return null; } if (StringUtils.isBlank(functionCallConfig.getName())) { @@ -158,21 +167,28 @@ public class PluginServiceImpl implements PluginService { List pluginList = getPluginList(); return pluginList.stream() - .filter(plugin -> { - PluginParseConfig functionCallConfig = getPluginParseConfig(plugin); - if (functionCallConfig == null) { - return false; - } - return true; - }) - .collect(Collectors.toMap(a -> { - PluginParseConfig functionCallConfig = JsonUtil.toObject( - a.getParseModeConfig(), PluginParseConfig.class); - return functionCallConfig.getName(); - }, a -> a, (k1, k2) -> k1)); + .filter( + plugin -> { + PluginParseConfig functionCallConfig = getPluginParseConfig(plugin); + if (functionCallConfig == null) { + return false; + } + return true; + }) + .collect( + Collectors.toMap( + a -> { + PluginParseConfig functionCallConfig = + JsonUtil.toObject( + a.getParseModeConfig(), + PluginParseConfig.class); + return functionCallConfig.getName(); + }, + a -> a, + (k1, k2) -> k1)); } - //todo + // todo private List authCheck(List plugins, User user) { return plugins; } @@ -181,8 +197,10 @@ public class PluginServiceImpl implements PluginService { ChatPlugin plugin = new ChatPlugin(); BeanUtils.copyProperties(pluginDO, plugin); if (StringUtils.isNotBlank(pluginDO.getDataSet())) { - plugin.setDataSetList(Arrays.stream(pluginDO.getDataSet().split(",")) - .map(Long::parseLong).collect(Collectors.toList())); + plugin.setDataSetList( + Arrays.stream(pluginDO.getDataSet().split(",")) + .map(Long::parseLong) + .collect(Collectors.toList())); } return plugin; } @@ -212,5 +230,4 @@ public class PluginServiceImpl implements PluginService { } return Lists.newArrayList(); } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java index 33e48fddb..238020a01 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.service.impl; import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; -import com.tencent.supersonic.chat.server.service.StatisticsService; import com.tencent.supersonic.chat.server.persistence.mapper.StatisticsMapper; +import com.tencent.supersonic.chat.server.service.StatisticsService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; @@ -14,8 +14,7 @@ import java.util.List; @Slf4j public class StatisticsServiceImpl implements StatisticsService { - @Autowired - private StatisticsMapper statisticsMapper; + @Autowired private StatisticsMapper statisticsMapper; @Async @Override diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java index 82f5cdb55..1e0024717 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java @@ -29,7 +29,6 @@ import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER; - @Component @Slf4j public class ChatConfigHelper { @@ -38,7 +37,10 @@ public class ChatConfigHelper { ChatConfig chatConfig = new ChatConfig(); BeanUtils.copyProperties(extendBaseCmd, chatConfig); RecordInfo recordInfo = new RecordInfo(); - String creator = (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) ? ADMIN_LOWER : user.getName(); + String creator = + (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) + ? ADMIN_LOWER + : user.getName(); recordInfo.createdBy(creator); chatConfig.setRecordInfo(recordInfo); chatConfig.setStatus(StatusEnum.ONLINE); @@ -50,8 +52,10 @@ public class ChatConfigHelper { BeanUtils.copyProperties(extendEditCmd, chatConfig); RecordInfo recordInfo = new RecordInfo(); - String user = (Objects.isNull(facadeUser) || StringUtils.isEmpty(facadeUser.getName())) - ? ADMIN_LOWER : facadeUser.getName(); + String user = + (Objects.isNull(facadeUser) || StringUtils.isEmpty(facadeUser.getName())) + ? ADMIN_LOWER + : facadeUser.getName(); recordInfo.updatedBy(user); chatConfig.setRecordInfo(recordInfo); return chatConfig; @@ -61,8 +65,9 @@ public class ChatConfigHelper { if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getDimensions())) { return new ArrayList<>(); } - Map> dimIdAndDescPair = modelSchema.getDimensions() - .stream().collect(Collectors.groupingBy(SchemaElement::getId)); + Map> dimIdAndDescPair = + modelSchema.getDimensions().stream() + .collect(Collectors.groupingBy(SchemaElement::getId)); return new ArrayList<>(dimIdAndDescPair.keySet()); } @@ -70,8 +75,9 @@ public class ChatConfigHelper { if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getMetrics())) { return new ArrayList<>(); } - Map> metricIdAndDescPair = modelSchema.getMetrics() - .stream().collect(Collectors.groupingBy(SchemaElement::getId)); + Map> metricIdAndDescPair = + modelSchema.getMetrics().stream() + .collect(Collectors.groupingBy(SchemaElement::getId)); return new ArrayList<>(metricIdAndDescPair.keySet()); } @@ -81,7 +87,8 @@ public class ChatConfigHelper { chatConfigDO.setChatAggConfig(JsonUtil.toString(chatConfig.getChatAggConfig())); chatConfigDO.setChatDetailConfig(JsonUtil.toString(chatConfig.getChatDetailConfig())); - chatConfigDO.setRecommendedQuestions(JsonUtil.toString(chatConfig.getRecommendedQuestions())); + chatConfigDO.setRecommendedQuestions( + JsonUtil.toString(chatConfig.getRecommendedQuestions())); if (Objects.isNull(chatConfig.getStatus())) { chatConfigDO.setStatus(null); @@ -112,7 +119,8 @@ public class ChatConfigHelper { chatConfigDescriptor.setChatAggConfig( JsonUtil.toObject(chatConfigDO.getChatAggConfig(), ChatAggConfigReq.class)); chatConfigDescriptor.setRecommendedQuestions( - JsonUtil.toList(chatConfigDO.getRecommendedQuestions(), RecommendedQuestionReq.class)); + JsonUtil.toList( + chatConfigDO.getRecommendedQuestions(), RecommendedQuestionReq.class)); chatConfigDescriptor.setStatusEnum(StatusEnum.of(chatConfigDO.getStatus())); chatConfigDescriptor.setCreatedBy(chatConfigDO.getCreatedBy()); @@ -120,7 +128,6 @@ public class ChatConfigHelper { chatConfigDescriptor.setUpdatedBy(chatConfigDO.getUpdatedBy()); chatConfigDescriptor.setUpdatedAt(chatConfigDO.getUpdatedAt()); - if (StringUtils.isEmpty(chatConfigDO.getChatAggConfig())) { chatConfigDescriptor.setChatAggConfig(generateEmptyChatAggConfigResp()); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java index ad43b7ac8..fc123e0d7 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java @@ -21,39 +21,45 @@ public class ComponentFactory { private static List pluginRecognizers = new ArrayList<>(); public static List getParseProcessors() { - return CollectionUtils.isEmpty(parseProcessors) ? init(ParseResultProcessor.class, - parseProcessors) : parseProcessors; + return CollectionUtils.isEmpty(parseProcessors) + ? init(ParseResultProcessor.class, parseProcessors) + : parseProcessors; } public static List getExecuteProcessors() { return CollectionUtils.isEmpty(executeProcessors) - ? init(ExecuteResultProcessor.class, executeProcessors) : executeProcessors; + ? init(ExecuteResultProcessor.class, executeProcessors) + : executeProcessors; } public static List getChatParsers() { return CollectionUtils.isEmpty(chatQueryParsers) - ? init(ChatQueryParser.class, chatQueryParsers) : chatQueryParsers; + ? init(ChatQueryParser.class, chatQueryParsers) + : chatQueryParsers; } public static List getChatExecutors() { return CollectionUtils.isEmpty(chatQueryExecutors) - ? init(ChatQueryExecutor.class, chatQueryExecutors) : chatQueryExecutors; + ? init(ChatQueryExecutor.class, chatQueryExecutors) + : chatQueryExecutors; } public static List getPluginRecognizers() { return CollectionUtils.isEmpty(pluginRecognizers) - ? init(PluginRecognizer.class, pluginRecognizers) : pluginRecognizers; + ? init(PluginRecognizer.class, pluginRecognizers) + : pluginRecognizers; } private static List init(Class factoryType, List list) { - list.addAll(SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader())); + list.addAll( + SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader())); return list; } private static T init(Class factoryType) { - return SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader()).get(0); + return SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader()) + .get(0); } - -} \ No newline at end of file +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java index 085fb20e6..fbe19a895 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.chat.server.util; import com.tencent.supersonic.chat.server.agent.Agent; +import com.tencent.supersonic.chat.server.pojo.ChatContext; import com.tencent.supersonic.chat.server.pojo.ParseContext; import com.tencent.supersonic.common.pojo.enums.Text2SQLType; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq; -import com.tencent.supersonic.chat.server.pojo.ChatContext; import org.apache.commons.collections.MapUtils; import java.util.Objects; @@ -49,5 +49,4 @@ public class QueryReqConverter { } return queryNLReq; } - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java index 820ceb1da..168a9084a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java @@ -8,7 +8,8 @@ import java.util.Map; public class ResultFormatter { - public static String transform2TextNew(List queryColumns, List> queryResults) { + public static String transform2TextNew( + List queryColumns, List> queryResults) { if (CollectionUtils.isEmpty(queryColumns)) { return ""; } @@ -35,5 +36,4 @@ public class ResultFormatter { } return table.toString(); } - } diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml deleted file mode 100644 index 59d0642a4..000000000 --- a/checkstyle/checkstyle.xml +++ /dev/null @@ -1,357 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java index ea8aaaa16..2a47bff7e 100644 --- a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java +++ b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java @@ -24,12 +24,13 @@ public class LoadRemoveService { } List resultList = new ArrayList<>(value); if (StringUtils.isNotBlank(mapperRemoveNaturePrefix)) { - resultList.removeIf(nature -> { - if (Objects.isNull(nature)) { - return false; - } - return nature.startsWith(mapperRemoveNaturePrefix); - }); + resultList.removeIf( + nature -> { + if (Objects.isNull(nature)) { + return false; + } + return nature.startsWith(mapperRemoveNaturePrefix); + }); } return resultList; } @@ -46,5 +47,4 @@ public class LoadRemoveService { } return null; } - } diff --git a/common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java b/common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java index e7b161db3..7ee3ab3e0 100644 --- a/common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java +++ b/common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java @@ -20,26 +20,16 @@ import java.util.Set; @Slf4j public abstract class BaseNode implements Comparable { - /** - * 状态数组，方便读取的时候用 - */ + /** 状态数组，方便读取的时候用 */ static final Status[] ARRAY_STATUS = Status.values(); - /** - * 子节点 - */ + /** 子节点 */ protected BaseNode[] child; - /** - * 节点状态 - */ + /** 节点状态 */ protected Status status; - /** - * 节点代表的字符 - */ + /** 节点代表的字符 */ protected char c; - /** - * 节点代表的值 - */ + /** 节点代表的值 */ protected V value; protected String prefix = null; @@ -238,25 +228,18 @@ public abstract class BaseNode implements Comparable { } public enum Status { - /** - * 未指定，用于删除词条 - */ + /** 未指定，用于删除词条 */ UNDEFINED_0, - /** - * 不是词语的结尾 - */ + /** 不是词语的结尾 */ NOT_WORD_1, - /** - * 是个词语的结尾，并且还可以继续 - */ + /** 是个词语的结尾，并且还可以继续 */ WORD_MIDDLE_2, - /** - * 是个词语的结尾，并且没有继续 - */ + /** 是个词语的结尾，并且没有继续 */ WORD_END_3, } - public class TrieEntry extends AbstractMap.SimpleEntry implements Comparable { + public class TrieEntry extends AbstractMap.SimpleEntry + implements Comparable { public TrieEntry(String key, V value) { super(key, value); @@ -295,8 +278,9 @@ public abstract class BaseNode implements Comparable { } } - /*** - * walk limit + /** + * * walk limit + * * @param sb * @param entrySet */ @@ -322,5 +306,4 @@ public abstract class BaseNode implements Comparable { } } } - } diff --git a/common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java b/common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java index 085e5d5ec..40d0f3bab 100644 --- a/common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java +++ b/common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java @@ -1,6 +1,5 @@ package com.hankcs.hanlp.dictionary; - import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.collection.trie.DoubleArrayTrie; import com.hankcs.hanlp.corpus.io.ByteArray; @@ -8,6 +7,7 @@ import com.hankcs.hanlp.corpus.io.IOUtil; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.utility.Predefine; import com.hankcs.hanlp.utility.TextUtility; + import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.DataOutputStream; @@ -21,9 +21,7 @@ import java.util.List; import java.util.TreeMap; import java.util.stream.Collectors; -/** - * 使用DoubleArrayTrie实现的核心词典 - */ +/** 使用DoubleArrayTrie实现的核心词典 */ public class CoreDictionary { public static DoubleArrayTrie trie = new DoubleArrayTrie(); @@ -36,8 +34,13 @@ public class CoreDictionary { if (!load(PATH)) { throw new IllegalArgumentException("核心词典" + PATH + "加载失败"); } else { - Predefine.logger.info(PATH + "加载成功，" + trie.size() + "个词条，耗时" - + (System.currentTimeMillis() - start) + "ms"); + Predefine.logger.info( + PATH + + "加载成功，" + + trie.size() + + "个词条，耗时" + + (System.currentTimeMillis() - start) + + "ms"); } } @@ -75,15 +78,21 @@ public class CoreDictionary { totalFrequency += attribute.totalFrequency; } Predefine.logger.info( - "核心词典读入词条" + map.size() + " 全部频次" + totalFrequency + "，耗时" + ( - System.currentTimeMillis() - start) + "核心词典读入词条" + + map.size() + + " 全部频次" + + totalFrequency + + "，耗时" + + (System.currentTimeMillis() - start) + "ms"); br.close(); trie.build(map); Predefine.logger.info("核心词典加载成功:" + trie.size() + "个词条，下面将写入缓存……"); try { - DataOutputStream out = new DataOutputStream( - new BufferedOutputStream(IOUtil.newOutputStream(path + Predefine.BIN_EXT))); + DataOutputStream out = + new DataOutputStream( + new BufferedOutputStream( + IOUtil.newOutputStream(path + Predefine.BIN_EXT))); Collection attributeList = map.values(); out.writeInt(attributeList.size()); for (Attribute attribute : attributeList) { @@ -202,25 +211,18 @@ public class CoreDictionary { return trie.get(key) != null; } - /** - * 核心词典中的词属性 - */ + /** 核心词典中的词属性 */ public static class Attribute implements Serializable { - /** - * 词性列表 - */ + /** 词性列表 */ public Nature[] nature; - /** - * 词性对应的词频 - */ + /** 词性对应的词频 */ public int[] frequency; public int totalFrequency; public String[] originals; public String original = null; - public Attribute(int size) { nature = new Nature[size]; frequency = new int[size]; @@ -276,8 +278,11 @@ public class CoreDictionary { } return attribute; } catch (Exception e) { - Predefine.logger.warning("使用字符串" + natureWithFrequency + "创建词条属性失败！" - + TextUtility.exceptionToString(e)); + Predefine.logger.warning( + "使用字符串" + + natureWithFrequency + + "创建词条属性失败！" + + TextUtility.exceptionToString(e)); return null; } } @@ -404,7 +409,10 @@ public class CoreDictionary { if (originals == null || originals.length == 0) { return null; } - return Arrays.stream(originals).filter(o -> o != null).distinct().collect(Collectors.toList()); + return Arrays.stream(originals) + .filter(o -> o != null) + .distinct() + .collect(Collectors.toList()); } } @@ -431,4 +439,3 @@ public class CoreDictionary { return load(path); } } - diff --git a/common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java b/common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java index 41d356e99..a5c16a92e 100644 --- a/common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java +++ b/common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java @@ -1,16 +1,15 @@ package com.hankcs.hanlp.seg; - import com.hankcs.hanlp.algorithm.Viterbi; import com.hankcs.hanlp.collection.AhoCorasick.AhoCorasickDoubleArrayTrie; import com.hankcs.hanlp.collection.trie.DoubleArrayTrie; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; -import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.dictionary.CoreDictionaryTransformMatrixDictionary; import com.hankcs.hanlp.dictionary.other.CharType; import com.hankcs.hanlp.seg.NShort.Path.AtomNode; import com.hankcs.hanlp.seg.common.Graph; +import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.seg.common.Vertex; import com.hankcs.hanlp.seg.common.WordNet; import com.hankcs.hanlp.utility.TextUtility; @@ -21,11 +20,9 @@ import java.util.LinkedList; import java.util.List; import java.util.ListIterator; - public abstract class WordBasedSegment extends Segment { - public WordBasedSegment() { - } + public WordBasedSegment() {} protected static void generateWord(List linkedArray, WordNet wordNetOptimum) { fixResultByRule(linkedArray); @@ -50,7 +47,9 @@ public abstract class WordBasedSegment extends Segment { } vertex = (Vertex) var1.next(); - } while (!vertex.realWord.equals("－－") && !vertex.realWord.equals("—") && !vertex.realWord.equals("-")); + } while (!vertex.realWord.equals("－－") + && !vertex.realWord.equals("—") + && !vertex.realWord.equals("-")); vertex.confirmNature(Nature.w); } @@ -64,9 +63,12 @@ public abstract class WordBasedSegment extends Segment { for (Vertex current = next; listIterator.hasNext(); current = next) { next = (Vertex) listIterator.next(); Nature currentNature = current.getNature(); - if (currentNature == Nature.nx && (next.hasNature(Nature.q) || next.hasNature(Nature.n))) { + if (currentNature == Nature.nx + && (next.hasNature(Nature.q) || next.hasNature(Nature.n))) { String[] param = current.realWord.split("-", 1); - if (param.length == 2 && TextUtility.isAllNum(param[0]) && TextUtility.isAllNum(param[1])) { + if (param.length == 2 + && TextUtility.isAllNum(param[0]) + && TextUtility.isAllNum(param[1])) { current = current.copy(); current.realWord = param[0]; current.confirmNature(Nature.m); @@ -79,7 +81,6 @@ public abstract class WordBasedSegment extends Segment { } } } - } } @@ -90,7 +91,8 @@ public abstract class WordBasedSegment extends Segment { for (Vertex current = next; listIterator.hasNext(); current = next) { next = (Vertex) listIterator.next(); - if (TextUtility.isAllNum(current.realWord) || TextUtility.isAllChineseNum(current.realWord)) { + if (TextUtility.isAllNum(current.realWord) + || TextUtility.isAllChineseNum(current.realWord)) { String nextWord = next.realWord; if (nextWord.length() == 1 && "月日时分秒".contains(nextWord) || nextWord.length() == 2 && nextWord.equals("月份")) { @@ -110,8 +112,10 @@ public abstract class WordBasedSegment extends Segment { current.confirmNature(Nature.m, true); } else if (current.realWord.length() > 1) { char last = current.realWord.charAt(current.realWord.length() - 1); - current = Vertex.newNumberInstance( - current.realWord.substring(0, current.realWord.length() - 1)); + current = + Vertex.newNumberInstance( + current.realWord.substring( + 0, current.realWord.length() - 1)); listIterator.previous(); listIterator.previous(); listIterator.set(current); @@ -121,7 +125,6 @@ public abstract class WordBasedSegment extends Segment { } } } - } } @@ -143,9 +146,7 @@ public abstract class WordBasedSegment extends Segment { return wordNet.toGraph(); } - /** - * @deprecated - */ + /** @deprecated */ private static List atomSegment(String sSentence, int start, int end) { if (end < start) { throw new RuntimeException("start=" + start + " < end=" + end); @@ -161,7 +162,10 @@ public abstract class WordBasedSegment extends Segment { charTypeArray[i] = CharType.get(c); if (c == '.' && i < charArray.length - 1 && CharType.get(charArray[i + 1]) == 9) { charTypeArray[i] = 9; - } else if (c == '.' && i < charArray.length - 1 && charArray[i + 1] >= '0' && charArray[i + 1] <= '9') { + } else if (c == '.' + && i < charArray.length - 1 + && charArray[i + 1] >= '0' + && charArray[i + 1] <= '9') { charTypeArray[i] = 5; } else if (charTypeArray[i] == 8) { charTypeArray[i] = 5; @@ -222,8 +226,10 @@ public abstract class WordBasedSegment extends Segment { while (true) { while (listIterator.hasNext()) { next = (Vertex) listIterator.next(); - if (!TextUtility.isAllNum(current.realWord) && !TextUtility.isAllChineseNum(current.realWord) - || !TextUtility.isAllNum(next.realWord) && !TextUtility.isAllChineseNum(next.realWord)) { + if (!TextUtility.isAllNum(current.realWord) + && !TextUtility.isAllChineseNum(current.realWord) + || !TextUtility.isAllNum(next.realWord) + && !TextUtility.isAllChineseNum(next.realWord)) { current = next; } else { current = Vertex.newNumberInstance(current.realWord + next.realWord); @@ -246,16 +252,24 @@ public abstract class WordBasedSegment extends Segment { DoubleArrayTrie.Searcher searcher = CoreDictionary.trie.getSearcher(charArray, 0); while (searcher.next()) { - wordNetStorage.add(searcher.begin + 1, new Vertex(new String(charArray, searcher.begin, searcher.length), - (CoreDictionary.Attribute) searcher.value, searcher.index)); + wordNetStorage.add( + searcher.begin + 1, + new Vertex( + new String(charArray, searcher.begin, searcher.length), + (CoreDictionary.Attribute) searcher.value, + searcher.index)); } if (this.config.forceCustomDictionary) { - this.customDictionary.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit() { - public void hit(int begin, int end, CoreDictionary.Attribute value) { - wordNetStorage.add(begin + 1, new Vertex(new String(charArray, begin, end - begin), value)); - } - }); + this.customDictionary.parseText( + charArray, + new AhoCorasickDoubleArrayTrie.IHit() { + public void hit(int begin, int end, CoreDictionary.Attribute value) { + wordNetStorage.add( + begin + 1, + new Vertex(new String(charArray, begin, end - begin), value)); + } + }); } LinkedList[] vertexes = wordNetStorage.getVertexes(); @@ -266,9 +280,10 @@ public abstract class WordBasedSegment extends Segment { if (vertexes[i].isEmpty()) { int j; for (j = i + 1; - j < vertexes.length - 1 && (vertexes[j].isEmpty() || CharType.get(charArray[j - 1]) == 11); - ++j) { - } + j < vertexes.length - 1 + && (vertexes[j].isEmpty() + || CharType.get(charArray[j - 1]) == 11); + ++j) {} wordNetStorage.add(i, Segment.quickAtomSegment(charArray, i - 1, j - 1)); i = j; @@ -291,12 +306,14 @@ public abstract class WordBasedSegment extends Segment { for (int i = 0; i < length; ++i) { Vertex vertex = (Vertex) listIterator.next(); Term termMain = Segment.convert(vertex); - //termList.add(termMain); + // termList.add(termMain); addTerms(termList, vertex, line - 1); termMain.offset = line - 1; if (vertex.realWord.length() > 2) { label43: - for (int currentLine = line; currentLine < line + vertex.realWord.length(); ++currentLine) { + for (int currentLine = line; + currentLine < line + vertex.realWord.length(); + ++currentLine) { Iterator iterator = wordNetAll.descendingIterator(currentLine); while (true) { @@ -310,11 +327,12 @@ public abstract class WordBasedSegment extends Segment { && smallVertex.realWord.length() < this.config.indexMode); if (smallVertex != vertex - && currentLine + smallVertex.realWord.length() <= line + vertex.realWord.length()) { + && currentLine + smallVertex.realWord.length() + <= line + vertex.realWord.length()) { listIterator.add(smallVertex); - //Term termSub = convert(smallVertex); - //termSub.offset = currentLine - 1; - //termList.add(termSub); + // Term termSub = convert(smallVertex); + // termSub.offset = currentLine - 1; + // termList.add(termSub); addTerms(termList, smallVertex, currentLine - 1); } } @@ -328,7 +346,8 @@ public abstract class WordBasedSegment extends Segment { } protected static void speechTagging(List vertexList) { - Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary); + Viterbi.compute( + vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary); } protected void addTerms(List terms, Vertex vertex, int offset) { diff --git a/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java b/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java index 78ec273e2..f98acc642 100644 --- a/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java +++ b/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java @@ -1,14 +1,9 @@ package com.hankcs.hanlp.seg.common; import com.hankcs.hanlp.corpus.tag.Nature; -//import com.hankcs.hanlp.dictionary.CoreDictionary; -//import com.hankcs.hanlp.dictionary.CustomDictionary; -//import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import lombok.Data; import lombok.ToString; -//import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; - @Data @ToString public class Term { @@ -72,5 +67,4 @@ public class Term { } return super.equals(obj); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/S2MysqlSqlDialect.java b/common/src/main/java/com/tencent/supersonic/common/calcite/S2MysqlSqlDialect.java index e379d1433..1fb70d94f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/S2MysqlSqlDialect.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/S2MysqlSqlDialect.java @@ -14,4 +14,4 @@ public class S2MysqlSqlDialect extends MysqlSqlDialect { buf.append(val.replace(this.literalEndQuoteString, this.literalEscapedQuote)); buf.append(this.literalEndQuoteString); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParseUtils.java b/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParseUtils.java index 517c7758a..42b040d87 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParseUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParseUtils.java @@ -1,13 +1,5 @@ package com.tencent.supersonic.common.calcite; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlIdentifier; @@ -25,9 +17,15 @@ import org.apache.calcite.sql.util.SqlString; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -/** - * sql parse utils - */ +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** sql parse utils */ public class SqlParseUtils { /** @@ -44,10 +42,13 @@ public class SqlParseUtils { handlerSQL(sqlNode, sqlParserInfo); - sqlParserInfo.setAllFields(sqlParserInfo.getAllFields().stream().distinct().collect(Collectors.toList())); + sqlParserInfo.setAllFields( + sqlParserInfo.getAllFields().stream().distinct().collect(Collectors.toList())); sqlParserInfo.setSelectFields( - sqlParserInfo.getSelectFields().stream().distinct().collect(Collectors.toList())); + sqlParserInfo.getSelectFields().stream() + .distinct() + .collect(Collectors.toList())); return sqlParserInfo; } catch (SqlParseException e) { @@ -107,16 +108,18 @@ public class SqlParseUtils { SqlSelect sqlSelect = (SqlSelect) select; SqlNodeList selectList = sqlSelect.getSelectList(); - selectList.getList().forEach(list -> { - Set selectFields = handlerField(list); - sqlParserInfo.getSelectFields().addAll(selectFields); - }); + selectList + .getList() + .forEach( + list -> { + Set selectFields = handlerField(list); + sqlParserInfo.getSelectFields().addAll(selectFields); + }); String tableName = handlerFrom(sqlSelect.getFrom()); sqlParserInfo.setTableName(tableName); Set selectFields = handlerSelectField(sqlSelect); allFields.addAll(selectFields); - } private static Set handlerSelectField(SqlSelect sqlSelect) { @@ -126,10 +129,14 @@ public class SqlParseUtils { results.addAll(formFields); } - sqlSelect.getSelectList().getList().forEach(list -> { - Set selectFields = handlerField(list); - results.addAll(selectFields); - }); + sqlSelect + .getSelectList() + .getList() + .forEach( + list -> { + Set selectFields = handlerField(list); + results.addAll(selectFields); + }); if (sqlSelect.hasWhere()) { Set whereFields = handlerField(sqlSelect.getWhere()); @@ -141,11 +148,11 @@ public class SqlParseUtils { } SqlNodeList group = sqlSelect.getGroup(); if (group != null) { - group.forEach(groupField -> { - Set groupByFields = handlerField(groupField); - results.addAll(groupByFields); - - }); + group.forEach( + groupField -> { + Set groupByFields = handlerField(groupField); + results.addAll(groupByFields); + }); } return results; } @@ -206,9 +213,12 @@ public class SqlParseUtils { } } if (field instanceof SqlNodeList) { - ((SqlNodeList) field).getList().forEach(node -> { - fields.addAll(handlerField(node)); - }); + ((SqlNodeList) field) + .getList() + .forEach( + node -> { + fields.addAll(handlerField(node)); + }); } break; } @@ -232,10 +242,14 @@ public class SqlParseUtils { if (CollectionUtils.isNotEmpty(operandList) && operandList.size() == 1) { SqlIdentifier sqlIdentifier = (SqlIdentifier) operandList.get(0); String simple = sqlIdentifier.getSimple(); - SqlBasicCall aliasedNode = new SqlBasicCall( - SqlStdOperatorTable.AS, - new SqlNode[]{sqlBasicCall, new SqlIdentifier(simple.toLowerCase(), SqlParserPos.ZERO)}, - SqlParserPos.ZERO); + SqlBasicCall aliasedNode = + new SqlBasicCall( + SqlStdOperatorTable.AS, + new SqlNode[] { + sqlBasicCall, + new SqlIdentifier(simple.toLowerCase(), SqlParserPos.ZERO) + }, + SqlParserPos.ZERO); selectList.set(selectList.indexOf(node), aliasedNode); } } @@ -245,7 +259,8 @@ public class SqlParseUtils { return newSql.getSql().replaceAll("`", ""); } - public static String addFieldsToSql(String sql, List addFields) throws SqlParseException { + public static String addFieldsToSql(String sql, List addFields) + throws SqlParseException { if (CollectionUtils.isEmpty(addFields)) { return sql; } @@ -335,8 +350,8 @@ public class SqlParseUtils { SqlNode sqlNodeCase = parser.parseExpression(); if (sqlNodeCase instanceof SqlCase) { SqlCase sqlCase = (SqlCase) sqlNodeCase; - if (CollectionUtils.isEmpty(sqlCase.getThenOperands()) || CollectionUtils.isEmpty( - sqlCase.getWhenOperands())) { + if (CollectionUtils.isEmpty(sqlCase.getThenOperands()) + || CollectionUtils.isEmpty(sqlCase.getWhenOperands())) { return ret; } SqlDialect dialect = new S2MysqlSqlDialect(S2MysqlSqlDialect.DEFAULT_CONTEXT); @@ -346,10 +361,12 @@ public class SqlParseUtils { SqlBasicCall when = (SqlBasicCall) sqlNode; if (!org.springframework.util.CollectionUtils.isEmpty(when.getOperandList()) && when.getOperandList().size() > 1) { - String value = when.getOperandList().get(1).toSqlString(dialect).getSql(); + String value = + when.getOperandList().get(1).toSqlString(dialect).getSql(); if (sqlCase.getThenOperands().get(i) != null) { if (sqlCase.getThenOperands().get(i) instanceof SqlIdentifier) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) sqlCase.getThenOperands().get(i); + SqlIdentifier sqlIdentifier = + (SqlIdentifier) sqlCase.getThenOperands().get(i); String field = sqlIdentifier.getSimple(); ret.put(value, field); } @@ -364,6 +381,4 @@ public class SqlParseUtils { } return ret; } - } - diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParserInfo.java b/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParserInfo.java index 318809577..b2f0cf125 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParserInfo.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/SqlParserInfo.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.common.calcite; +import lombok.Data; +import lombok.ToString; + import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import lombok.Data; -import lombok.ToString; @Data @ToString @@ -15,4 +16,4 @@ public class SqlParserInfo implements Serializable { private List selectFields = new ArrayList<>(); private List allFields = new ArrayList<>(); -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/config/ChatModelParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/ChatModelParameterConfig.java index 75e710ac2..4d9aa6124 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/ChatModelParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/ChatModelParameterConfig.java @@ -21,55 +21,97 @@ import java.util.List; public class ChatModelParameterConfig extends ParameterConfig { public static final Parameter CHAT_MODEL_PROVIDER = - new Parameter("s2.chat.model.provider", OpenAiModelFactory.PROVIDER, - "接口协议", "", "list", - "对话模型配置", getCandidateValues()); + new Parameter( + "s2.chat.model.provider", + OpenAiModelFactory.PROVIDER, + "接口协议", + "", + "list", + "对话模型配置", + getCandidateValues()); public static final Parameter CHAT_MODEL_BASE_URL = - new Parameter("s2.chat.model.base.url", OpenAiModelFactory.DEFAULT_BASE_URL, - "BaseUrl", "", "string", - "对话模型配置", null, getBaseUrlDependency()); + new Parameter( + "s2.chat.model.base.url", + OpenAiModelFactory.DEFAULT_BASE_URL, + "BaseUrl", + "", + "string", + "对话模型配置", + null, + getBaseUrlDependency()); public static final Parameter CHAT_MODEL_ENDPOINT = - new Parameter("s2.chat.model.endpoint", "llama_2_70b", - "Endpoint", "", "string", - "对话模型配置", null, getEndpointDependency()); + new Parameter( + "s2.chat.model.endpoint", + "llama_2_70b", + "Endpoint", + "", + "string", + "对话模型配置", + null, + getEndpointDependency()); public static final Parameter CHAT_MODEL_API_KEY = - new Parameter("s2.chat.model.api.key", DEMO, - "ApiKey", "", "password", - "对话模型配置", null, getApiKeyDependency() - ); + new Parameter( + "s2.chat.model.api.key", + DEMO, + "ApiKey", + "", + "password", + "对话模型配置", + null, + getApiKeyDependency()); public static final Parameter CHAT_MODEL_SECRET_KEY = - new Parameter("s2.chat.model.secretKey", "demo", - "SecretKey", "", "password", - "对话模型配置", null, getSecretKeyDependency()); + new Parameter( + "s2.chat.model.secretKey", + "demo", + "SecretKey", + "", + "password", + "对话模型配置", + null, + getSecretKeyDependency()); public static final Parameter CHAT_MODEL_NAME = - new Parameter("s2.chat.model.name", "gpt-3.5-turbo", - "ModelName", "", "string", - "对话模型配置", null, getModelNameDependency()); + new Parameter( + "s2.chat.model.name", + "gpt-3.5-turbo", + "ModelName", + "", + "string", + "对话模型配置", + null, + getModelNameDependency()); public static final Parameter CHAT_MODEL_ENABLE_SEARCH = - new Parameter("s2.chat.model.enableSearch", "false", - "是否启用搜索增强功能，设为false表示不启用", "", "bool", - "对话模型配置", null, getEnableSearchDependency()); + new Parameter( + "s2.chat.model.enableSearch", + "false", + "是否启用搜索增强功能，设为false表示不启用", + "", + "bool", + "对话模型配置", + null, + getEnableSearchDependency()); public static final Parameter CHAT_MODEL_TEMPERATURE = - new Parameter("s2.chat.model.temperature", "0.0", - "Temperature", "", - "slider", "对话模型配置"); + new Parameter( + "s2.chat.model.temperature", "0.0", "Temperature", "", "slider", "对话模型配置"); public static final Parameter CHAT_MODEL_TIMEOUT = - new Parameter("s2.chat.model.timeout", "60", - "超时时间(秒)", "", - "number", "对话模型配置"); + new Parameter("s2.chat.model.timeout", "60", "超时时间(秒)", "", "number", "对话模型配置"); @Override public List getSysParameters() { return Lists.newArrayList( - CHAT_MODEL_PROVIDER, CHAT_MODEL_BASE_URL, CHAT_MODEL_ENDPOINT, - CHAT_MODEL_API_KEY, CHAT_MODEL_SECRET_KEY, CHAT_MODEL_NAME, - CHAT_MODEL_ENABLE_SEARCH, CHAT_MODEL_TEMPERATURE, CHAT_MODEL_TIMEOUT - ); + CHAT_MODEL_PROVIDER, + CHAT_MODEL_BASE_URL, + CHAT_MODEL_ENDPOINT, + CHAT_MODEL_API_KEY, + CHAT_MODEL_SECRET_KEY, + CHAT_MODEL_NAME, + CHAT_MODEL_ENABLE_SEARCH, + CHAT_MODEL_TEMPERATURE, + CHAT_MODEL_TIMEOUT); } public ChatModelConfig convert() { @@ -108,7 +150,8 @@ public class ChatModelParameterConfig extends ParameterConfig { } private static List getBaseUrlDependency() { - return getDependency(CHAT_MODEL_PROVIDER.getName(), + return getDependency( + CHAT_MODEL_PROVIDER.getName(), getCandidateValues(), ImmutableMap.of( OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL, @@ -117,32 +160,31 @@ public class ChatModelParameterConfig extends ParameterConfig { QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_BASE_URL, ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_BASE_URL, LocalAiModelFactory.PROVIDER, LocalAiModelFactory.DEFAULT_BASE_URL, - DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_BASE_URL) - ); + DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_BASE_URL)); } private static List getApiKeyDependency() { - return getDependency(CHAT_MODEL_PROVIDER.getName(), + return getDependency( + CHAT_MODEL_PROVIDER.getName(), Lists.newArrayList( OpenAiModelFactory.PROVIDER, QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER, LocalAiModelFactory.PROVIDER, AzureModelFactory.PROVIDER, - DashscopeModelFactory.PROVIDER - ), + DashscopeModelFactory.PROVIDER), ImmutableMap.of( OpenAiModelFactory.PROVIDER, DEMO, QianfanModelFactory.PROVIDER, DEMO, ZhipuModelFactory.PROVIDER, DEMO, LocalAiModelFactory.PROVIDER, DEMO, AzureModelFactory.PROVIDER, DEMO, - DashscopeModelFactory.PROVIDER, DEMO - )); + DashscopeModelFactory.PROVIDER, DEMO)); } private static List getModelNameDependency() { - return getDependency(CHAT_MODEL_PROVIDER.getName(), + return getDependency( + CHAT_MODEL_PROVIDER.getName(), getCandidateValues(), ImmutableMap.of( OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_MODEL_NAME, @@ -151,29 +193,28 @@ public class ChatModelParameterConfig extends ParameterConfig { ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_MODEL_NAME, LocalAiModelFactory.PROVIDER, LocalAiModelFactory.DEFAULT_MODEL_NAME, AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_MODEL_NAME, - DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_MODEL_NAME - ) - ); + DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_MODEL_NAME)); } private static List getEndpointDependency() { - return getDependency(CHAT_MODEL_PROVIDER.getName(), + return getDependency( + CHAT_MODEL_PROVIDER.getName(), Lists.newArrayList(QianfanModelFactory.PROVIDER), - ImmutableMap.of(QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_ENDPOINT) - ); + ImmutableMap.of( + QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_ENDPOINT)); } private static List getEnableSearchDependency() { - return getDependency(CHAT_MODEL_PROVIDER.getName(), + return getDependency( + CHAT_MODEL_PROVIDER.getName(), Lists.newArrayList(DashscopeModelFactory.PROVIDER), - ImmutableMap.of(DashscopeModelFactory.PROVIDER, "false") - ); + ImmutableMap.of(DashscopeModelFactory.PROVIDER, "false")); } private static List getSecretKeyDependency() { - return getDependency(CHAT_MODEL_PROVIDER.getName(), + return getDependency( + CHAT_MODEL_PROVIDER.getName(), Lists.newArrayList(QianfanModelFactory.PROVIDER), - ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO) - ); + ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO)); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/DataBaseConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/DataBaseConfig.java index bf3441313..3e53ca8fe 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/DataBaseConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/DataBaseConfig.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.common.config; +import javax.sql.DataSource; + import com.alibaba.druid.pool.DruidDataSource; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import javax.sql.DataSource; - @Configuration @Primary public class DataBaseConfig { @@ -21,5 +21,4 @@ public class DataBaseConfig { druidDataSource.setValidationQuery("select 1"); return druidDataSource; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java index 3d1134605..13f322165 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java @@ -32,5 +32,4 @@ public class EmbeddingConfig { public String getMemoryCollectionName(Integer agentId) { return memoryCollectionPrefix + agentId; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingModelParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingModelParameterConfig.java index 54d65afa8..7158cd280 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingModelParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingModelParameterConfig.java @@ -22,46 +22,89 @@ import java.util.List; @Slf4j public class EmbeddingModelParameterConfig extends ParameterConfig { public static final Parameter EMBEDDING_MODEL_PROVIDER = - new Parameter("s2.embedding.model.provider", InMemoryModelFactory.PROVIDER, - "接口协议", "", "list", - "向量模型配置", getCandidateValues()); + new Parameter( + "s2.embedding.model.provider", + InMemoryModelFactory.PROVIDER, + "接口协议", + "", + "list", + "向量模型配置", + getCandidateValues()); public static final Parameter EMBEDDING_MODEL_BASE_URL = - new Parameter("s2.embedding.model.base.url", "", - "BaseUrl", "", "string", - "向量模型配置", null, getBaseUrlDependency() - ); + new Parameter( + "s2.embedding.model.base.url", + "", + "BaseUrl", + "", + "string", + "向量模型配置", + null, + getBaseUrlDependency()); public static final Parameter EMBEDDING_MODEL_API_KEY = - new Parameter("s2.embedding.model.api.key", "", - "ApiKey", "", "password", - "向量模型配置", null, getApiKeyDependency()); + new Parameter( + "s2.embedding.model.api.key", + "", + "ApiKey", + "", + "password", + "向量模型配置", + null, + getApiKeyDependency()); public static final Parameter EMBEDDING_MODEL_SECRET_KEY = - new Parameter("s2.embedding.model.secretKey", "demo", - "SecretKey", "", "password", - "向量模型配置", null, getSecretKeyDependency()); + new Parameter( + "s2.embedding.model.secretKey", + "demo", + "SecretKey", + "", + "password", + "向量模型配置", + null, + getSecretKeyDependency()); public static final Parameter EMBEDDING_MODEL_NAME = - new Parameter("s2.embedding.model.name", EmbeddingModelConstant.BGE_SMALL_ZH, - "ModelName", "", "string", - "向量模型配置", null, getModelNameDependency()); + new Parameter( + "s2.embedding.model.name", + EmbeddingModelConstant.BGE_SMALL_ZH, + "ModelName", + "", + "string", + "向量模型配置", + null, + getModelNameDependency()); public static final Parameter EMBEDDING_MODEL_PATH = - new Parameter("s2.embedding.model.path", "", - "模型路径", "", "string", - "向量模型配置", null, getModelPathDependency()); + new Parameter( + "s2.embedding.model.path", + "", + "模型路径", + "", + "string", + "向量模型配置", + null, + getModelPathDependency()); public static final Parameter EMBEDDING_MODEL_VOCABULARY_PATH = - new Parameter("s2.embedding.model.vocabulary.path", "", - "词汇表路径", "", "string", - "向量模型配置", null, getModelPathDependency()); + new Parameter( + "s2.embedding.model.vocabulary.path", + "", + "词汇表路径", + "", + "string", + "向量模型配置", + null, + getModelPathDependency()); @Override public List getSysParameters() { return Lists.newArrayList( - EMBEDDING_MODEL_PROVIDER, EMBEDDING_MODEL_BASE_URL, EMBEDDING_MODEL_API_KEY, - EMBEDDING_MODEL_SECRET_KEY, EMBEDDING_MODEL_NAME, EMBEDDING_MODEL_PATH, - EMBEDDING_MODEL_VOCABULARY_PATH - ); + EMBEDDING_MODEL_PROVIDER, + EMBEDDING_MODEL_BASE_URL, + EMBEDDING_MODEL_API_KEY, + EMBEDDING_MODEL_SECRET_KEY, + EMBEDDING_MODEL_NAME, + EMBEDDING_MODEL_PATH, + EMBEDDING_MODEL_VOCABULARY_PATH); } public EmbeddingModelConfig convert() { @@ -91,13 +134,14 @@ public class EmbeddingModelParameterConfig extends ParameterConfig { AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER, - ZhipuModelFactory.PROVIDER - ); + ZhipuModelFactory.PROVIDER); } private static List getBaseUrlDependency() { - return getDependency(EMBEDDING_MODEL_PROVIDER.getName(), - Lists.newArrayList(OpenAiModelFactory.PROVIDER, + return getDependency( + EMBEDDING_MODEL_PROVIDER.getName(), + Lists.newArrayList( + OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER, AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER, @@ -109,28 +153,34 @@ public class EmbeddingModelParameterConfig extends ParameterConfig { AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_BASE_URL, DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_BASE_URL, QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_BASE_URL, - ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_BASE_URL - ) - ); + ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_BASE_URL)); } private static List getApiKeyDependency() { - return getDependency(EMBEDDING_MODEL_PROVIDER.getName(), - Lists.newArrayList(OpenAiModelFactory.PROVIDER, + return getDependency( + EMBEDDING_MODEL_PROVIDER.getName(), + Lists.newArrayList( + OpenAiModelFactory.PROVIDER, AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER), - ImmutableMap.of(OpenAiModelFactory.PROVIDER, DEMO, - AzureModelFactory.PROVIDER, DEMO, - DashscopeModelFactory.PROVIDER, DEMO, - QianfanModelFactory.PROVIDER, DEMO, - ZhipuModelFactory.PROVIDER, DEMO) - ); + ImmutableMap.of( + OpenAiModelFactory.PROVIDER, + DEMO, + AzureModelFactory.PROVIDER, + DEMO, + DashscopeModelFactory.PROVIDER, + DEMO, + QianfanModelFactory.PROVIDER, + DEMO, + ZhipuModelFactory.PROVIDER, + DEMO)); } private static List getModelNameDependency() { - return getDependency(EMBEDDING_MODEL_PROVIDER.getName(), + return getDependency( + EMBEDDING_MODEL_PROVIDER.getName(), Lists.newArrayList( InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER, @@ -138,31 +188,33 @@ public class EmbeddingModelParameterConfig extends ParameterConfig { AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER, - ZhipuModelFactory.PROVIDER - ), + ZhipuModelFactory.PROVIDER), ImmutableMap.of( InMemoryModelFactory.PROVIDER, EmbeddingModelConstant.BGE_SMALL_ZH, - OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, - OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, + OpenAiModelFactory.PROVIDER, + OpenAiModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, + OllamaModelFactory.PROVIDER, + OllamaModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, - DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, - QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, - ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_EMBEDDING_MODEL_NAME - ) - ); + DashscopeModelFactory.PROVIDER, + DashscopeModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, + QianfanModelFactory.PROVIDER, + QianfanModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, + ZhipuModelFactory.PROVIDER, + ZhipuModelFactory.DEFAULT_EMBEDDING_MODEL_NAME)); } private static List getModelPathDependency() { - return getDependency(EMBEDDING_MODEL_PROVIDER.getName(), + return getDependency( + EMBEDDING_MODEL_PROVIDER.getName(), Lists.newArrayList(InMemoryModelFactory.PROVIDER), - ImmutableMap.of(InMemoryModelFactory.PROVIDER, "") - ); + ImmutableMap.of(InMemoryModelFactory.PROVIDER, "")); } private static List getSecretKeyDependency() { - return getDependency(EMBEDDING_MODEL_PROVIDER.getName(), + return getDependency( + EMBEDDING_MODEL_PROVIDER.getName(), Lists.newArrayList(QianfanModelFactory.PROVIDER), - ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO) - ); + ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO)); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java index 9ab1d74d5..7284fe189 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java @@ -16,47 +16,82 @@ import java.util.List; @Slf4j public class EmbeddingStoreParameterConfig extends ParameterConfig { public static final Parameter EMBEDDING_STORE_PROVIDER = - new Parameter("s2.embedding.store.provider", EmbeddingStoreType.IN_MEMORY.name(), - "向量库类型", "目前支持三种类型：IN_MEMORY、MILVUS、CHROMA", "list", - "向量库配置", getCandidateValues()); + new Parameter( + "s2.embedding.store.provider", + EmbeddingStoreType.IN_MEMORY.name(), + "向量库类型", + "目前支持三种类型：IN_MEMORY、MILVUS、CHROMA", + "list", + "向量库配置", + getCandidateValues()); public static final Parameter EMBEDDING_STORE_BASE_URL = - new Parameter("s2.embedding.store.base.url", "", - "BaseUrl", "", "string", - "向量库配置", null, getBaseUrlDependency()); + new Parameter( + "s2.embedding.store.base.url", + "", + "BaseUrl", + "", + "string", + "向量库配置", + null, + getBaseUrlDependency()); public static final Parameter EMBEDDING_STORE_API_KEY = - new Parameter("s2.embedding.store.api.key", "", - "ApiKey", "", "password", - "向量库配置", null, getApiKeyDependency()); + new Parameter( + "s2.embedding.store.api.key", + "", + "ApiKey", + "", + "password", + "向量库配置", + null, + getApiKeyDependency()); public static final Parameter EMBEDDING_STORE_PERSIST_PATH = - new Parameter("s2.embedding.store.persist.path", "", - "持久化路径", "默认不持久化，如需持久化请填写持久化路径。" - + "注意：如果变更了向量模型需删除该路径下已保存的文件或修改持久化路径", "string", - "向量库配置", null, getPathDependency()); + new Parameter( + "s2.embedding.store.persist.path", + "", + "持久化路径", + "默认不持久化，如需持久化请填写持久化路径。" + "注意：如果变更了向量模型需删除该路径下已保存的文件或修改持久化路径", + "string", + "向量库配置", + null, + getPathDependency()); public static final Parameter EMBEDDING_STORE_TIMEOUT = - new Parameter("s2.embedding.store.timeout", "60", - "超时时间(秒)", "", - "number", "向量库配置"); + new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", "向量库配置"); public static final Parameter EMBEDDING_STORE_DIMENSION = - new Parameter("s2.embedding.store.dimension", "", - "纬度", "", "number", - "向量库配置", null, getDimensionDependency()); + new Parameter( + "s2.embedding.store.dimension", + "", + "纬度", + "", + "number", + "向量库配置", + null, + getDimensionDependency()); public static final Parameter EMBEDDING_STORE_DATABASE_NAME = - new Parameter("s2.embedding.store.databaseName", "", - "DatabaseName", "", "string", - "向量库配置", null, getDatabaseNameDependency()); + new Parameter( + "s2.embedding.store.databaseName", + "", + "DatabaseName", + "", + "string", + "向量库配置", + null, + getDatabaseNameDependency()); @Override public List getSysParameters() { return Lists.newArrayList( - EMBEDDING_STORE_PROVIDER, EMBEDDING_STORE_BASE_URL, EMBEDDING_STORE_API_KEY, - EMBEDDING_STORE_DATABASE_NAME, EMBEDDING_STORE_PERSIST_PATH, - EMBEDDING_STORE_TIMEOUT, EMBEDDING_STORE_DIMENSION - ); + EMBEDDING_STORE_PROVIDER, + EMBEDDING_STORE_BASE_URL, + EMBEDDING_STORE_API_KEY, + EMBEDDING_STORE_DATABASE_NAME, + EMBEDDING_STORE_PERSIST_PATH, + EMBEDDING_STORE_TIMEOUT, + EMBEDDING_STORE_DIMENSION); } public EmbeddingStoreConfig convert() { @@ -70,9 +105,15 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { if (StringUtils.isNumeric(getParameterValue(EMBEDDING_STORE_DIMENSION))) { dimension = Integer.valueOf(getParameterValue(EMBEDDING_STORE_DIMENSION)); } - return EmbeddingStoreConfig.builder().provider(provider).baseUrl(baseUrl) - .apiKey(apiKey).persistPath(persistPath).databaseName(databaseName) - .timeOut(Long.valueOf(timeOut)).dimension(dimension).build(); + return EmbeddingStoreConfig.builder() + .provider(provider) + .baseUrl(baseUrl) + .apiKey(apiKey) + .persistPath(persistPath) + .databaseName(databaseName) + .timeOut(Long.valueOf(timeOut)) + .dimension(dimension) + .build(); } private static ArrayList getCandidateValues() { @@ -83,40 +124,40 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { } private static List getBaseUrlDependency() { - return getDependency(EMBEDDING_STORE_PROVIDER.getName(), + return getDependency( + EMBEDDING_STORE_PROVIDER.getName(), Lists.newArrayList( EmbeddingStoreType.MILVUS.name(), EmbeddingStoreType.CHROMA.name()), ImmutableMap.of( EmbeddingStoreType.MILVUS.name(), "http://localhost:19530", - EmbeddingStoreType.CHROMA.name(), "http://localhost:8000" - ) - ); + EmbeddingStoreType.CHROMA.name(), "http://localhost:8000")); } private static List getApiKeyDependency() { - return getDependency(EMBEDDING_STORE_PROVIDER.getName(), + return getDependency( + EMBEDDING_STORE_PROVIDER.getName(), Lists.newArrayList(EmbeddingStoreType.MILVUS.name()), - ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO) - ); + ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO)); } private static List getPathDependency() { - return getDependency(EMBEDDING_STORE_PROVIDER.getName(), + return getDependency( + EMBEDDING_STORE_PROVIDER.getName(), Lists.newArrayList(EmbeddingStoreType.IN_MEMORY.name()), ImmutableMap.of(EmbeddingStoreType.IN_MEMORY.name(), "")); } private static List getDimensionDependency() { - return getDependency(EMBEDDING_STORE_PROVIDER.getName(), + return getDependency( + EMBEDDING_STORE_PROVIDER.getName(), Lists.newArrayList(EmbeddingStoreType.MILVUS.name()), - ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "384") - ); + ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "384")); } private static List getDatabaseNameDependency() { - return getDependency(EMBEDDING_STORE_PROVIDER.getName(), + return getDependency( + EMBEDDING_STORE_PROVIDER.getName(), Lists.newArrayList(EmbeddingStoreType.MILVUS.name()), - ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "") - ); + ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "")); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/config/JdbcTemplateConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/JdbcTemplateConfig.java index 698182eaf..1e8e986c7 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/JdbcTemplateConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/JdbcTemplateConfig.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.common.config; +import javax.sql.DataSource; + import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jdbc.core.JdbcTemplate; -import javax.sql.DataSource; - @Configuration public class JdbcTemplateConfig { @@ -14,5 +14,4 @@ public class JdbcTemplateConfig { public JdbcTemplate jdbcTemplate(@Qualifier("h2") DataSource dataSource) { return new JdbcTemplate(dataSource); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/ParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/ParameterConfig.java index d1d8ab40c..f74b319a1 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/ParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/ParameterConfig.java @@ -15,24 +15,19 @@ import java.util.Map; @Service public abstract class ParameterConfig { public static final String DEMO = "demo"; - @Autowired - private SystemConfigService sysConfigService; + @Autowired private SystemConfigService sysConfigService; - @Autowired - private Environment environment; + @Autowired private Environment environment; - /** - * @return system parameters to be set with user interface - */ + /** @return system parameters to be set with user interface */ protected List getSysParameters() { return Collections.EMPTY_LIST; } /** - * Parameter value will be derived in the following order: - * 1. `system config` set with user interface - * 2. `system property` set with application.yaml file - * 3. `default value` set with parameter declaration + * Parameter value will be derived in the following order: 1. `system config` set with user + * interface 2. `system property` set with application.yaml file 3. `default value` set with + * parameter declaration * * @param parameter instance * @return parameter value @@ -67,5 +62,4 @@ public abstract class ParameterConfig { dependencies.add(dependency); return dependencies; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java index bf3d8d6e7..9cf839b05 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java @@ -10,5 +10,4 @@ import lombok.NoArgsConstructor; public class PromptConfig { private String promptTemplate; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/SystemConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/SystemConfig.java index 4645ae62d..1599346a3 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/SystemConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/SystemConfig.java @@ -38,8 +38,11 @@ public class SystemConfig { if (StringUtils.isBlank(name)) { return ""; } - Map nameToValue = getParameters().stream() - .collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (k1, k2) -> k1)); + Map nameToValue = + getParameters().stream() + .collect( + Collectors.toMap( + Parameter::getName, Parameter::getValue, (k1, k2) -> k1)); return nameToValue.get(name); } @@ -66,13 +69,16 @@ public class SystemConfig { if (CollectionUtils.isEmpty(parameters)) { return defaultParameters; } - Map parameterNameValueMap = parameters.stream() - .collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (v1, v2) -> v2)); + Map parameterNameValueMap = + parameters.stream() + .collect( + Collectors.toMap( + Parameter::getName, Parameter::getValue, (v1, v2) -> v2)); for (Parameter parameter : defaultParameters) { - parameter.setValue(parameterNameValueMap.getOrDefault(parameter.getName(), - parameter.getDefaultValue())); + parameter.setValue( + parameterNameValueMap.getOrDefault( + parameter.getName(), parameter.getDefaultValue())); } return defaultParameters; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/ThreadContextConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/ThreadContextConfig.java index 54af52630..978c4e51e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/ThreadContextConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/ThreadContextConfig.java @@ -11,4 +11,4 @@ public class ThreadContextConfig { public S2ThreadContext s2ThreadContext() { return new S2ThreadContext(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/config/VisualConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/VisualConfig.java index c912b36df..0bb152639 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/VisualConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/VisualConfig.java @@ -8,5 +8,4 @@ public class VisualConfig { private boolean enableSimpleMode; private boolean showDebugInfo; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/WebMvcConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/WebMvcConfig.java index 39ba053c7..c7f7eddf9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/WebMvcConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/WebMvcConfig.java @@ -13,4 +13,3 @@ public class WebMvcConfig implements WebMvcConfigurer { registry.addInterceptor(new LogInterceptor()).addPathPatterns("/**"); } } - diff --git a/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java b/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java index 69324b181..b18c92b93 100644 --- a/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java +++ b/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.common.interceptor; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.common.util.TraceIdUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -7,15 +10,13 @@ import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; import org.springframework.web.servlet.ModelAndView; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - @Component @Slf4j public class LogInterceptor implements HandlerInterceptor { @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { - //use previous traceId + public boolean preHandle( + HttpServletRequest request, HttpServletResponse response, Object handler) { + // use previous traceId String traceId = request.getHeader(TraceIdUtil.TRACE_ID); if (StringUtils.isBlank(traceId)) { TraceIdUtil.setTraceId(TraceIdUtil.generateTraceId()); @@ -26,16 +27,18 @@ public class LogInterceptor implements HandlerInterceptor { } @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, - Object handler, ModelAndView modelAndView) - throws Exception { - - } + public void postHandle( + HttpServletRequest request, + HttpServletResponse response, + Object handler, + ModelAndView modelAndView) + throws Exception {} @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) + public void afterCompletion( + HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { - //remove after Completing + // remove after Completing TraceIdUtil.remove(); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/AggregateEnum.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/AggregateEnum.java index 12a595813..8f0cdb6e5 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/AggregateEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/AggregateEnum.java @@ -30,6 +30,8 @@ public enum AggregateEnum { public static Map getAggregateEnum() { return Arrays.stream(AggregateEnum.values()) - .collect(Collectors.toMap(AggregateEnum::getAggregateCh, AggregateEnum::getAggregateEN)); + .collect( + Collectors.toMap( + AggregateEnum::getAggregateCh, AggregateEnum::getAggregateEN)); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateFunctionHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateFunctionHelper.java index 6f6aacb01..aad1e73a9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateFunctionHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateFunctionHelper.java @@ -13,7 +13,8 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList; @Slf4j public class DateFunctionHelper { - public static String getStartDateStr(ComparisonOperator minorThanEquals, ExpressionList expressions) { + public static String getStartDateStr( + ComparisonOperator minorThanEquals, ExpressionList expressions) { String unitValue = getUnit(expressions); String dateValue = getEndDateValue(expressions); String dateStr = ""; @@ -22,9 +23,11 @@ public class DateFunctionHelper { if (rightExpression instanceof DoubleValue) { DoubleValue value = (DoubleValue) rightExpression; double doubleValue = value.getValue(); - if (DatePeriodEnum.YEAR.equals(datePeriodEnum) && doubleValue == JsqlConstants.HALF_YEAR) { + if (DatePeriodEnum.YEAR.equals(datePeriodEnum) + && doubleValue == JsqlConstants.HALF_YEAR) { datePeriodEnum = DatePeriodEnum.MONTH; - dateStr = DateUtils.getBeforeDate(dateValue, JsqlConstants.SIX_MONTH, datePeriodEnum); + dateStr = + DateUtils.getBeforeDate(dateValue, JsqlConstants.SIX_MONTH, datePeriodEnum); } } else if (rightExpression instanceof LongValue) { LongValue value = (LongValue) rightExpression; @@ -47,5 +50,4 @@ public class DateFunctionHelper { StringValue unit = (StringValue) expressions.get(0); return unit.getValue(); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateVisitor.java index 933b961e5..06467ec23 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/DateVisitor.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.List; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; @@ -12,6 +11,8 @@ import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; import net.sf.jsqlparser.schema.Column; import org.springframework.util.CollectionUtils; +import java.util.List; + @Slf4j public class DateVisitor extends ExpressionVisitorAdapter { diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ExpressionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ExpressionReplaceVisitor.java index ad216fe9d..7670e926e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ExpressionReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ExpressionReplaceVisitor.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.Map; -import java.util.Objects; import net.sf.jsqlparser.expression.BinaryExpression; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; @@ -10,11 +8,13 @@ import net.sf.jsqlparser.expression.WhenClause; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; import net.sf.jsqlparser.schema.Column; +import java.util.Map; +import java.util.Objects; + public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter { private Map fieldExprMap; - public ExpressionReplaceVisitor(Map fieldExprMap) { this.fieldExprMap = fieldExprMap; } @@ -23,8 +23,9 @@ public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter { expr.getWhenExpression().accept(this); if (expr.getThenExpression() instanceof Column) { Column column = (Column) expr.getThenExpression(); - Expression expression = QueryExpressionReplaceVisitor.getExpression( - QueryExpressionReplaceVisitor.getReplaceExpr(column, fieldExprMap)); + Expression expression = + QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr(column, fieldExprMap)); if (Objects.nonNull(expression)) { expr.setThenExpression(expression); } @@ -51,16 +52,20 @@ public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter { } } if (left instanceof Column) { - Expression expression = QueryExpressionReplaceVisitor.getExpression( - QueryExpressionReplaceVisitor.getReplaceExpr((Column) left, fieldExprMap)); + Expression expression = + QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr( + (Column) left, fieldExprMap)); if (Objects.nonNull(expression)) { expr.setLeftExpression(expression); leftVisited = true; } } if (right instanceof Column) { - Expression expression = QueryExpressionReplaceVisitor.getExpression( - QueryExpressionReplaceVisitor.getReplaceExpr((Column) right, fieldExprMap)); + Expression expression = + QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr( + (Column) right, fieldExprMap)); if (Objects.nonNull(expression)) { expr.setRightExpression(expression); rightVisited = true; @@ -76,8 +81,9 @@ public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter { private boolean visitFunction(Function function) { if (function.getParameters().getExpressions().get(0) instanceof Column) { - Expression expression = QueryExpressionReplaceVisitor.getExpression( - QueryExpressionReplaceVisitor.getReplaceExpr(function, fieldExprMap)); + Expression expression = + QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr(function, fieldExprMap)); if (Objects.nonNull(expression)) { ExpressionList expressions = new ExpressionList<>(); expressions.add(expression); diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAcquireVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAcquireVisitor.java index bcca801e8..15393afec 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAcquireVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAcquireVisitor.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.Set; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; import net.sf.jsqlparser.schema.Column; +import java.util.Set; + public class FieldAcquireVisitor extends ExpressionVisitorAdapter { private Set fields; @@ -17,4 +18,4 @@ public class FieldAcquireVisitor extends ExpressionVisitorAdapter { String columnName = column.getColumnName(); fields.add(columnName); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAndValueAcquireVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAndValueAcquireVisitor.java index 532740455..0f12865d9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAndValueAcquireVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldAndValueAcquireVisitor.java @@ -1,12 +1,6 @@ package com.tencent.supersonic.common.jsqlparser; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import net.sf.jsqlparser.expression.DoubleValue; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; @@ -25,6 +19,13 @@ import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; import net.sf.jsqlparser.schema.Column; import org.apache.commons.collections.CollectionUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter { private Set fieldExpressions; @@ -124,14 +125,16 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter { fieldExpression.setFieldName(field.getColumnName()); fieldExpression.setFunction(functionName); fieldExpression.setOperator(expr.getStringExpression()); - //deal with DAY/WEEK function - List collect = Arrays.stream(DatePeriodEnum.values()).collect(Collectors.toList()); + // deal with DAY/WEEK function + List collect = + Arrays.stream(DatePeriodEnum.values()).collect(Collectors.toList()); DatePeriodEnum periodEnum = DatePeriodEnum.get(functionName); if (Objects.nonNull(periodEnum) && collect.contains(periodEnum)) { - fieldExpression.setFieldValue(getFieldValue(rightExpression) + periodEnum.getChName()); + fieldExpression.setFieldValue( + getFieldValue(rightExpression) + periodEnum.getChName()); return fieldExpression; } else { - //deal with aggregate function + // deal with aggregate function fieldExpression.setFieldValue(getFieldValue(rightExpression)); return fieldExpression; } @@ -142,7 +145,8 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter { } private Column getColumn(Function leftExpressionFunction) { - //List leftExpressions = leftExpressionFunction.getParameters().getExpressions(); + // List leftExpressions = + // leftExpressionFunction.getParameters().getExpressions(); ExpressionList leftExpressions = leftExpressionFunction.getParameters(); if (CollectionUtils.isEmpty(leftExpressions)) { return null; diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldExpression.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldExpression.java index e37a0f5e9..02c96cd88 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldExpression.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldExpression.java @@ -12,5 +12,4 @@ public class FieldExpression { private Object fieldValue; private String function; - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldReplaceVisitor.java index 66ba6a084..39aefbe89 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldReplaceVisitor.java @@ -33,4 +33,4 @@ public class FieldReplaceVisitor extends ExpressionVisitorAdapter { exactReplace.set(originalExactReplace); } } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java index 11f070e8f..c19db1633 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java @@ -31,7 +31,8 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter { private boolean exactReplace; private Map> filedNameToValueMap; - public FieldValueReplaceVisitor(boolean exactReplace, Map> filedNameToValueMap) { + public FieldValueReplaceVisitor( + boolean exactReplace, Map> filedNameToValueMap) { this.exactReplace = exactReplace; this.filedNameToValueMap = filedNameToValueMap; } @@ -66,20 +67,24 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter { ExpressionList rightItemsList = (ExpressionList) inExpression.getRightExpression(); List expressions = rightItemsList.getExpressions(); List values = new ArrayList<>(); - expressions.stream().forEach(o -> { - if (o instanceof StringValue) { - values.add(((StringValue) o).getValue()); - } - }); + expressions.stream() + .forEach( + o -> { + if (o instanceof StringValue) { + values.add(((StringValue) o).getValue()); + } + }); if (valueMap == null || CollectionUtils.isEmpty(values)) { return; } List newExpressions = new ArrayList<>(); - values.stream().forEach(o -> { - String replaceValue = valueMap.getOrDefault(o, o); - StringValue stringValue = new StringValue(replaceValue); - newExpressions.add(stringValue); - }); + values.stream() + .forEach( + o -> { + String replaceValue = valueMap.getOrDefault(o, o); + StringValue stringValue = new StringValue(replaceValue); + newExpressions.add(stringValue); + }); rightItemsList.setExpressions(newExpressions); inExpression.setRightExpression(rightItemsList); } @@ -107,21 +112,24 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter { } if (rightExpression instanceof LongValue) { LongValue rightStringValue = (LongValue) rightExpression; - String replaceValue = getReplaceValue(valueMap, String.valueOf(rightStringValue.getValue())); + String replaceValue = + getReplaceValue(valueMap, String.valueOf(rightStringValue.getValue())); if (StringUtils.isNotEmpty(replaceValue)) { rightStringValue.setValue(Long.parseLong(replaceValue)); } } if (rightExpression instanceof DoubleValue) { DoubleValue rightStringValue = (DoubleValue) rightExpression; - String replaceValue = getReplaceValue(valueMap, String.valueOf(rightStringValue.getValue())); + String replaceValue = + getReplaceValue(valueMap, String.valueOf(rightStringValue.getValue())); if (StringUtils.isNotEmpty(replaceValue)) { rightStringValue.setValue(Double.parseDouble(replaceValue)); } } if (rightExpression instanceof StringValue) { StringValue rightStringValue = (StringValue) rightExpression; - String replaceValue = getReplaceValue(valueMap, String.valueOf(rightStringValue.getValue())); + String replaceValue = + getReplaceValue(valueMap, String.valueOf(rightStringValue.getValue())); if (StringUtils.isNotEmpty(replaceValue)) { rightStringValue.setValue(replaceValue); } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledExpression.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledExpression.java index 56394ce65..8fce0c939 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledExpression.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledExpression.java @@ -8,5 +8,4 @@ public class FiledExpression { private String operator; private String fieldName; - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java index d04b300e5..448ac06a4 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java @@ -65,7 +65,8 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter { @Override public void visit(GreaterThanEquals expr) { - List expressions = parserFilter(expr, JsqlConstants.GREATER_THAN_EQUALS_CONSTANT); + List expressions = + parserFilter(expr, JsqlConstants.GREATER_THAN_EQUALS_CONSTANT); if (Objects.nonNull(expressions)) { waitingForAdds.addAll(expressions); } @@ -101,7 +102,8 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter { } try { - ComparisonOperator parsedExpression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr); + ComparisonOperator parsedExpression = + (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr); comparisonOperator.setLeftExpression(parsedExpression.getLeftExpression()); comparisonOperator.setRightExpression(parsedExpression.getRightExpression()); comparisonOperator.setASTNode(parsedExpression.getASTNode()); diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledNameReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledNameReplaceVisitor.java index d5e2bbc7b..3d4fe836f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledNameReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledNameReplaceVisitor.java @@ -34,9 +34,11 @@ public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter { Expression leftExpression = expr.getLeftExpression(); Expression rightExpression = expr.getRightExpression(); - if (!(rightExpression instanceof StringValue) || !(leftExpression instanceof Column) + if (!(rightExpression instanceof StringValue) + || !(leftExpression instanceof Column) || CollectionUtils.isEmpty(fieldValueToFieldNames) - || Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) { + || Objects.isNull(rightExpression) + || Objects.isNull(leftExpression)) { return; } @@ -44,9 +46,9 @@ public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter { StringValue rightStringValue = (StringValue) rightExpression; Set fieldNames = fieldValueToFieldNames.get(rightStringValue.getValue()); - if (!CollectionUtils.isEmpty(fieldNames) && !fieldNames.contains(leftColumn.getColumnName())) { + if (!CollectionUtils.isEmpty(fieldNames) + && !fieldNames.contains(leftColumn.getColumnName())) { leftColumn.setColumnName(fieldNames.stream().findFirst().get()); } } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionAliasReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionAliasReplaceVisitor.java index d35b84e0c..00e8aa9ce 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionAliasReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionAliasReplaceVisitor.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.common.jsqlparser; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.statement.select.SelectItem; +import net.sf.jsqlparser.statement.select.SelectItemVisitorAdapter; + import java.util.HashMap; import java.util.Map; import java.util.Objects; -import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.statement.select.SelectItem; -import net.sf.jsqlparser.statement.select.SelectItemVisitorAdapter; public class FunctionAliasReplaceVisitor extends SelectItemVisitorAdapter { @@ -16,11 +17,12 @@ public class FunctionAliasReplaceVisitor extends SelectItemVisitorAdapter { if (selectExpressionItem.getExpression() instanceof Function) { Function function = (Function) selectExpressionItem.getExpression(); String columnName = SqlSelectHelper.getColumnName(function); - //1.exist alias. as - //2.alias's fieldName not equal. "sum(pv) as pv" cannot be replaced. - if (Objects.nonNull(selectExpressionItem.getAlias()) && !selectExpressionItem.getAlias().getName() - .equalsIgnoreCase(columnName)) { - aliasToActualExpression.put(selectExpressionItem.getAlias().getName(), function.toString()); + // 1.exist alias. as + // 2.alias's fieldName not equal. "sum(pv) as pv" cannot be replaced. + if (Objects.nonNull(selectExpressionItem.getAlias()) + && !selectExpressionItem.getAlias().getName().equalsIgnoreCase(columnName)) { + aliasToActualExpression.put( + selectExpressionItem.getAlias().getName(), function.toString()); selectExpressionItem.setAlias(null); } } @@ -29,4 +31,4 @@ public class FunctionAliasReplaceVisitor extends SelectItemVisitorAdapter { public Map getAliasToActualExpression() { return aliasToActualExpression; } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionNameReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionNameReplaceVisitor.java index 3cb93dc14..fca4d7965 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionNameReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionNameReplaceVisitor.java @@ -16,7 +16,8 @@ public class FunctionNameReplaceVisitor extends ExpressionVisitorAdapter { private Map functionMap; private Map functionCallMap; - public FunctionNameReplaceVisitor(Map functionMap, Map functionCallMap) { + public FunctionNameReplaceVisitor( + Map functionMap, Map functionCallMap) { this.functionMap = functionMap; this.functionCallMap = functionCallMap; } @@ -35,5 +36,4 @@ public class FunctionNameReplaceVisitor extends ExpressionVisitorAdapter { } } } - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionVisitor.java index 7369e0648..58b10c35b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionVisitor.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.common.jsqlparser; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.Function; + import java.util.HashSet; import java.util.Set; -import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; -import net.sf.jsqlparser.expression.Function; public class FunctionVisitor extends ExpressionVisitorAdapter { @@ -18,4 +19,4 @@ public class FunctionVisitor extends ExpressionVisitorAdapter { super.visit(function); functionNames.add(function.getName()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByFunctionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByFunctionReplaceVisitor.java index 0f9f57334..ffad895d0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByFunctionReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByFunctionReplaceVisitor.java @@ -19,7 +19,8 @@ public class GroupByFunctionReplaceVisitor implements GroupByVisitor { private Map functionMap; private Map functionCallMap; - public GroupByFunctionReplaceVisitor(Map functionMap, Map functionCallMap) { + public GroupByFunctionReplaceVisitor( + Map functionMap, Map functionCallMap) { this.functionMap = functionMap; this.functionCallMap = functionCallMap; } @@ -48,4 +49,4 @@ public class GroupByFunctionReplaceVisitor implements GroupByVisitor { } } } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByReplaceVisitor.java index 0a01ed06d..f63d88af2 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByReplaceVisitor.java @@ -35,7 +35,8 @@ public class GroupByReplaceVisitor implements GroupByVisitor { Expression expression = groupByExpressions.get(i); String columnName = getColumnName(expression); - String replaceColumn = parseVisitorHelper.getReplaceValue(columnName, fieldNameMap, exactReplace); + String replaceColumn = + parseVisitorHelper.getReplaceValue(columnName, fieldNameMap, exactReplace); if (StringUtils.isNotEmpty(replaceColumn)) { replaceExpression(groupByExpressions, i, expression, replaceColumn); } @@ -52,10 +53,11 @@ public class GroupByReplaceVisitor implements GroupByVisitor { return expression.toString(); } - private void replaceExpression(List groupByExpressions, - int index, - Expression expression, - String replaceColumn) { + private void replaceExpression( + List groupByExpressions, + int index, + Expression expression, + String replaceColumn) { if (expression instanceof Column) { groupByExpressions.set(index, new Column(replaceColumn)); } else if (expression instanceof Function) { @@ -66,9 +68,9 @@ public class GroupByReplaceVisitor implements GroupByVisitor { Function function = (Function) expression; if (function.getParameters().size() > 1) { - function.getParameters().stream().skip(1).forEach( - e -> newExpressionList.add((Function) e) - ); + function.getParameters().stream() + .skip(1) + .forEach(e -> newExpressionList.add((Function) e)); } function.setParameters(newExpressionList); } catch (JSQLParserException e) { @@ -76,4 +78,4 @@ public class GroupByReplaceVisitor implements GroupByVisitor { } } } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByVisitor.java index 6c5f638fb..cccf81dc1 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByVisitor.java @@ -15,4 +15,4 @@ public class GroupByVisitor implements net.sf.jsqlparser.statement.select.GroupB public boolean isHasAggregateFunction() { return hasAggregateFunction; } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/JsqlConstants.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/JsqlConstants.java index 8ad918e6f..3d2cff49a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/JsqlConstants.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/JsqlConstants.java @@ -27,19 +27,26 @@ public class JsqlConstants { public static final String IN_CONSTANT = " 1 in (1) "; public static final String LIKE_CONSTANT = "1 like 1"; public static final String IN = "IN"; - public static final Map rightMap = Stream.of( - new AbstractMap.SimpleEntry<>("<=", "<="), - new AbstractMap.SimpleEntry<>("<", "<"), - new AbstractMap.SimpleEntry<>(">=", "<="), - new AbstractMap.SimpleEntry<>(">", "<"), - new AbstractMap.SimpleEntry<>("=", "<=")) - .collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue)); - public static final Map leftMap = Stream.of( - new AbstractMap.SimpleEntry<>("<=", ">="), - new AbstractMap.SimpleEntry<>("<", ">"), - new AbstractMap.SimpleEntry<>(">=", "<="), - new AbstractMap.SimpleEntry<>(">", "<"), - new AbstractMap.SimpleEntry<>("=", ">=")) - .collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue)); - + public static final Map rightMap = + Stream.of( + new AbstractMap.SimpleEntry<>("<=", "<="), + new AbstractMap.SimpleEntry<>("<", "<"), + new AbstractMap.SimpleEntry<>(">=", "<="), + new AbstractMap.SimpleEntry<>(">", "<"), + new AbstractMap.SimpleEntry<>("=", "<=")) + .collect( + toMap( + AbstractMap.SimpleEntry::getKey, + AbstractMap.SimpleEntry::getValue)); + public static final Map leftMap = + Stream.of( + new AbstractMap.SimpleEntry<>("<=", ">="), + new AbstractMap.SimpleEntry<>("<", ">"), + new AbstractMap.SimpleEntry<>(">=", "<="), + new AbstractMap.SimpleEntry<>(">", "<"), + new AbstractMap.SimpleEntry<>("=", ">=")) + .collect( + toMap( + AbstractMap.SimpleEntry::getKey, + AbstractMap.SimpleEntry::getValue)); } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByAcquireVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByAcquireVisitor.java index 4a6a1209b..867264f74 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByAcquireVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByAcquireVisitor.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.jsqlparser; import com.tencent.supersonic.common.pojo.Constants; -import java.util.Set; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; @@ -9,6 +8,8 @@ import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.OrderByElement; import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter; +import java.util.Set; + public class OrderByAcquireVisitor extends OrderByVisitorAdapter { private Set fields; @@ -26,7 +27,7 @@ public class OrderByAcquireVisitor extends OrderByVisitorAdapter { } if (expression instanceof Function) { Function function = (Function) expression; - //List expressions = function.getParameters().getExpressions(); + // List expressions = function.getParameters().getExpressions(); ExpressionList expressions = function.getParameters(); for (Expression column : expressions) { if (column instanceof Column) { @@ -42,4 +43,4 @@ public class OrderByAcquireVisitor extends OrderByVisitorAdapter { fields.add(fieldExpression); super.visit(orderBy); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByExpression.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByExpression.java index 6ab6db4a9..64674c1b6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByExpression.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByExpression.java @@ -12,5 +12,4 @@ public class OrderByExpression { private Object fieldValue; private String function; - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByReplaceVisitor.java index af07d61c8..58b5466a4 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/OrderByReplaceVisitor.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.Map; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; @@ -8,6 +7,8 @@ import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.OrderByElement; import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter; +import java.util.Map; + public class OrderByReplaceVisitor extends OrderByVisitorAdapter { ParseVisitorHelper parseVisitorHelper = new ParseVisitorHelper(); @@ -27,7 +28,7 @@ public class OrderByReplaceVisitor extends OrderByVisitorAdapter { } if (expression instanceof Function) { Function function = (Function) expression; - //List expressions = function.getParameters().getExpressions(); + // List expressions = function.getParameters().getExpressions(); ExpressionList expressions = function.getParameters(); for (Expression column : expressions) { if (column instanceof Column) { @@ -37,4 +38,4 @@ public class OrderByReplaceVisitor extends OrderByVisitorAdapter { } super.visit(orderBy); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ParseVisitorHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ParseVisitorHelper.java index b07496de8..6b9970a42 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ParseVisitorHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/ParseVisitorHelper.java @@ -1,17 +1,19 @@ package com.tencent.supersonic.common.jsqlparser; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.schema.Column; +import org.apache.commons.lang3.StringUtils; + import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.schema.Column; -import org.apache.commons.lang3.StringUtils; @Slf4j public class ParseVisitorHelper { - public void replaceColumn(Column column, Map fieldNameMap, boolean exactReplace) { + public void replaceColumn( + Column column, Map fieldNameMap, boolean exactReplace) { String columnName = column.getColumnName(); String replaceColumn = getReplaceValue(columnName, fieldNameMap, exactReplace); if (StringUtils.isNotBlank(replaceColumn)) { @@ -19,7 +21,8 @@ public class ParseVisitorHelper { } } - public String getReplaceValue(String beforeValue, Map valueMap, boolean exactReplace) { + public String getReplaceValue( + String beforeValue, Map valueMap, boolean exactReplace) { String value = valueMap.get(beforeValue); if (StringUtils.isNotBlank(value)) { return value; @@ -27,13 +30,19 @@ public class ParseVisitorHelper { if (exactReplace) { return null; } - Optional> first = valueMap.entrySet().stream().sorted((k1, k2) -> { - String k1Value = k1.getKey(); - String k2Value = k2.getKey(); - Double k1Similarity = getSimilarity(beforeValue, k1Value); - Double k2Similarity = getSimilarity(beforeValue, k2Value); - return k2Similarity.compareTo(k1Similarity); - }).collect(Collectors.toList()).stream().findFirst(); + Optional> first = + valueMap.entrySet().stream() + .sorted( + (k1, k2) -> { + String k1Value = k1.getKey(); + String k2Value = k2.getKey(); + Double k1Similarity = getSimilarity(beforeValue, k1Value); + Double k2Similarity = getSimilarity(beforeValue, k2Value); + return k2Similarity.compareTo(k1Similarity); + }) + .collect(Collectors.toList()) + .stream() + .findFirst(); if (first.isPresent()) { return first.get().getValue(); @@ -58,10 +67,16 @@ public class ParseVisitorHelper { char cj = word2.charAt(j - 1); if (ci == cj) { dp[i][j] = dp[i - 1][j - 1]; - } else if (i > 1 && j > 1 && ci == word2.charAt(j - 2) && cj == word1.charAt(i - 2)) { + } else if (i > 1 + && j > 1 + && ci == word2.charAt(j - 2) + && cj == word1.charAt(i - 2)) { dp[i][j] = 1 + Math.min(dp[i - 2][j - 2], Math.min(dp[i][j - 1], dp[i - 1][j])); } else { - dp[i][j] = Math.min(dp[i - 1][j - 1] + 1, Math.min(dp[i][j - 1] + 1, dp[i - 1][j] + 1)); + dp[i][j] = + Math.min( + dp[i - 1][j - 1] + 1, + Math.min(dp[i][j - 1] + 1, dp[i - 1][j] + 1)); } } } @@ -71,4 +86,4 @@ public class ParseVisitorHelper { public double getSimilarity(String word1, String word2) { return 1 - (double) editDistance(word1, word2) / Math.max(word2.length(), word1.length()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java index bbe340f23..f47d99bf4 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.Map; -import java.util.Objects; import net.sf.jsqlparser.expression.Alias; import net.sf.jsqlparser.expression.BinaryExpression; import net.sf.jsqlparser.expression.Expression; @@ -11,6 +9,9 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.SelectItem; +import java.util.Map; +import java.util.Objects; + public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { private Map fieldExprMap; @@ -40,7 +41,6 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { } expr.getLeftExpression().accept(this); expr.getRightExpression().accept(this); - } public void visit(SelectItem selectExpressionItem) { @@ -55,7 +55,6 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { columnName = column.getColumnName(); toReplace = getReplaceExpr(leftFunc, fieldExprMap); } - } if (expression instanceof Column) { Column column = (Column) expression; @@ -103,12 +102,13 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { } public static String getReplaceExpr(Column column, Map fieldExprMap) { - return fieldExprMap.containsKey(column.getColumnName()) ? fieldExprMap.get(column.getColumnName()) : ""; + return fieldExprMap.containsKey(column.getColumnName()) + ? fieldExprMap.get(column.getColumnName()) + : ""; } public static String getReplaceExpr(Function function, Map fieldExprMap) { Column column = (Column) function.getParameters().getExpressions().get(0); return getReplaceExpr(column, fieldExprMap); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java index f2c7181d9..fe18d6cb8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java @@ -1,12 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.ArrayList; -import java.util.HashSet; - import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; @@ -22,18 +15,23 @@ import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.statement.select.GroupByElement; import net.sf.jsqlparser.statement.select.OrderByElement; +import net.sf.jsqlparser.statement.select.ParenthesedSelect; import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.statement.select.SelectItem; import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; import net.sf.jsqlparser.statement.select.SetOperationList; -import net.sf.jsqlparser.statement.select.ParenthesedSelect; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; -/** - * Sql Parser add Helper - */ +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** Sql Parser add Helper */ @Slf4j public class SqlAddHelper { @@ -45,21 +43,32 @@ public class SqlAddHelper { } if (selectStatement instanceof PlainSelect) { PlainSelect plainSelect = (PlainSelect) selectStatement; - fields.stream().filter(Objects::nonNull).forEach(field -> { - SelectItem selectExpressionItem = new SelectItem(new Column(field)); - plainSelect.addSelectItems(selectExpressionItem); - }); + fields.stream() + .filter(Objects::nonNull) + .forEach( + field -> { + SelectItem selectExpressionItem = + new SelectItem(new Column(field)); + plainSelect.addSelectItems(selectExpressionItem); + }); } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - fields.stream().forEach(field -> { - SelectItem selectExpressionItem = new SelectItem(new Column(field)); - subPlainSelect.addSelectItems(selectExpressionItem); - }); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + fields.stream() + .forEach( + field -> { + SelectItem selectExpressionItem = + new SelectItem(new Column(field)); + subPlainSelect.addSelectItems( + selectExpressionItem); + }); + }); } } return selectStatement.toString(); @@ -76,12 +85,16 @@ public class SqlAddHelper { PlainSelect plainSelect = (PlainSelect) selectStatement.getPlainSelect(); plainSelectList.add(plainSelect); } else if (selectStatement instanceof SetOperationList) { - SetOperationList setOperationList = (SetOperationList) selectStatement.getSetOperationList(); + SetOperationList setOperationList = + (SetOperationList) selectStatement.getSetOperationList(); if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - plainSelectList.add(subPlainSelect); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + }); } } @@ -225,15 +238,18 @@ public class SqlAddHelper { if (!(selectStatement instanceof PlainSelect)) { return sql; } - selectStatement.accept(new SelectVisitorAdapter() { - @Override - public void visit(PlainSelect plainSelect) { - addAggregateToSelectItems(plainSelect.getSelectItems(), fieldNameToAggregate); - addAggregateToOrderByItems(plainSelect.getOrderByElements(), fieldNameToAggregate); - addAggregateToGroupByItems(plainSelect.getGroupBy(), fieldNameToAggregate); - addAggregateToWhereItems(plainSelect.getWhere(), fieldNameToAggregate); - } - }); + selectStatement.accept( + new SelectVisitorAdapter() { + @Override + public void visit(PlainSelect plainSelect) { + addAggregateToSelectItems( + plainSelect.getSelectItems(), fieldNameToAggregate); + addAggregateToOrderByItems( + plainSelect.getOrderByElements(), fieldNameToAggregate); + addAggregateToGroupByItems(plainSelect.getGroupBy(), fieldNameToAggregate); + addAggregateToWhereItems(plainSelect.getWhere(), fieldNameToAggregate); + } + }); return selectStatement.toString(); } @@ -260,11 +276,12 @@ public class SqlAddHelper { return selectStatement.toString(); } - private static void addAggregateToSelectItems(List> selectItems, - Map fieldNameToAggregate) { + private static void addAggregateToSelectItems( + List> selectItems, Map fieldNameToAggregate) { for (SelectItem selectItem : selectItems) { Expression expression = selectItem.getExpression(); - Function function = SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); + Function function = + SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); if (function == null) { continue; } @@ -272,14 +289,15 @@ public class SqlAddHelper { } } - private static void addAggregateToOrderByItems(List orderByElements, - Map fieldNameToAggregate) { + private static void addAggregateToOrderByItems( + List orderByElements, Map fieldNameToAggregate) { if (orderByElements == null) { return; } for (OrderByElement orderByElement : orderByElements) { Expression expression = orderByElement.getExpression(); - Function function = SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); + Function function = + SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); if (function == null) { continue; } @@ -287,14 +305,15 @@ public class SqlAddHelper { } } - private static void addAggregateToGroupByItems(GroupByElement groupByElement, - Map fieldNameToAggregate) { + private static void addAggregateToGroupByItems( + GroupByElement groupByElement, Map fieldNameToAggregate) { if (groupByElement == null) { return; } for (int i = 0; i < groupByElement.getGroupByExpressionList().size(); i++) { Expression expression = (Expression) groupByElement.getGroupByExpressionList().get(i); - Function function = SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); + Function function = + SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); if (function == null) { continue; } @@ -302,15 +321,16 @@ public class SqlAddHelper { } } - private static void addAggregateToWhereItems(Expression whereExpression, Map fieldNameToAggregate) { + private static void addAggregateToWhereItems( + Expression whereExpression, Map fieldNameToAggregate) { if (whereExpression == null) { return; } modifyWhereExpression(whereExpression, fieldNameToAggregate); } - private static void modifyWhereExpression(Expression whereExpression, - Map fieldNameToAggregate) { + private static void modifyWhereExpression( + Expression whereExpression, Map fieldNameToAggregate) { if (SqlSelectHelper.isLogicExpression(whereExpression)) { if (whereExpression instanceof AndExpression) { AndExpression andExpression = (AndExpression) whereExpression; @@ -327,29 +347,36 @@ public class SqlAddHelper { modifyWhereExpression(rightExpression, fieldNameToAggregate); } } else if (whereExpression instanceof Parenthesis) { - modifyWhereExpression(((Parenthesis) whereExpression).getExpression(), fieldNameToAggregate); + modifyWhereExpression( + ((Parenthesis) whereExpression).getExpression(), fieldNameToAggregate); } else { setAggToFunction(whereExpression, fieldNameToAggregate); } } - private static void setAggToFunction(Expression expression, Map fieldNameToAggregate) { + private static void setAggToFunction( + Expression expression, Map fieldNameToAggregate) { if (!(expression instanceof ComparisonOperator)) { return; } ComparisonOperator comparisonOperator = (ComparisonOperator) expression; if (comparisonOperator.getRightExpression() instanceof Column) { - String columnName = ((Column) (comparisonOperator).getRightExpression()).getColumnName(); - Function function = SqlSelectFunctionHelper.getFunction(comparisonOperator.getRightExpression(), - fieldNameToAggregate.get(columnName)); + String columnName = + ((Column) (comparisonOperator).getRightExpression()).getColumnName(); + Function function = + SqlSelectFunctionHelper.getFunction( + comparisonOperator.getRightExpression(), + fieldNameToAggregate.get(columnName)); if (Objects.nonNull(function)) { comparisonOperator.setRightExpression(function); } } if (comparisonOperator.getLeftExpression() instanceof Column) { String columnName = ((Column) (comparisonOperator).getLeftExpression()).getColumnName(); - Function function = SqlSelectFunctionHelper.getFunction(comparisonOperator.getLeftExpression(), - fieldNameToAggregate.get(columnName)); + Function function = + SqlSelectFunctionHelper.getFunction( + comparisonOperator.getLeftExpression(), + fieldNameToAggregate.get(columnName)); if (Objects.nonNull(function)) { comparisonOperator.setLeftExpression(function); } @@ -364,7 +391,7 @@ public class SqlAddHelper { } PlainSelect plainSelect = (PlainSelect) selectStatement; - //replace metric to 1 and 1 and add having metric + // replace metric to 1 and 1 and add having metric Expression where = plainSelect.getWhere(); FiledFilterReplaceVisitor visitor = new FiledFilterReplaceVisitor(fieldNames); if (Objects.nonNull(where)) { @@ -423,4 +450,3 @@ public class SqlAddHelper { return selectStatement.toString(); } } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java index 838572832..1cf70c5b0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java @@ -1,16 +1,14 @@ package com.tencent.supersonic.common.jsqlparser; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; - -import java.util.List; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.statement.select.PlainSelect; -/** - * Date field parsing helper class - */ +import java.util.List; +import java.util.Objects; + +/** Date field parsing helper class */ @Slf4j public class SqlDateSelectHelper { @@ -32,4 +30,3 @@ public class SqlDateSelectHelper { return dateVisitor.getDateBoundInfo(); } } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java index fd13570fd..ce5d63df3 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java @@ -36,9 +36,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; -/** - * Sql Parser remove Helper - */ +/** Sql Parser remove Helper */ @Slf4j public class SqlRemoveHelper { @@ -67,14 +65,15 @@ public class SqlRemoveHelper { } List> selectItems = ((PlainSelect) selectStatement).getSelectItems(); Set fields = new HashSet<>(); - selectItems.removeIf(selectItem -> { - String field = selectItem.getExpression().toString(); - if (fields.contains(field)) { - return true; - } - fields.add(field); - return false; - }); + selectItems.removeIf( + selectItem -> { + String field = selectItem.getExpression().toString(); + if (fields.contains(field)) { + return true; + } + fields.add(field); + return false; + }); ((PlainSelect) selectStatement).setSelectItems(selectItems); return selectStatement.toString(); } @@ -84,16 +83,18 @@ public class SqlRemoveHelper { if (!(selectStatement instanceof PlainSelect)) { return sql; } - selectStatement.accept(new SelectVisitorAdapter() { - @Override - public void visit(PlainSelect plainSelect) { - removeWhereCondition(plainSelect.getWhere(), removeFieldNames); - } - }); + selectStatement.accept( + new SelectVisitorAdapter() { + @Override + public void visit(PlainSelect plainSelect) { + removeWhereCondition(plainSelect.getWhere(), removeFieldNames); + } + }); return removeNumberFilter(selectStatement.toString()); } - private static void removeWhereCondition(Expression whereExpression, Set removeFieldNames) { + private static void removeWhereCondition( + Expression whereExpression, Set removeFieldNames) { if (whereExpression == null) { return; } @@ -108,15 +109,18 @@ public class SqlRemoveHelper { Expression where = ((PlainSelect) selectStatement).getWhere(); Expression having = ((PlainSelect) selectStatement).getHaving(); try { - ((PlainSelect) selectStatement).setWhere(filteredExpression(where, SqlEditEnum.NUMBER_FILTER)); - ((PlainSelect) selectStatement).setHaving(filteredExpression(having, SqlEditEnum.NUMBER_FILTER)); + ((PlainSelect) selectStatement) + .setWhere(filteredExpression(where, SqlEditEnum.NUMBER_FILTER)); + ((PlainSelect) selectStatement) + .setHaving(filteredExpression(having, SqlEditEnum.NUMBER_FILTER)); } catch (Exception e) { log.info("replaceFunction has an exception:{}", e.toString()); } return selectStatement.toString(); } - private static void removeWhereExpression(Expression whereExpression, Set removeFieldNames) { + private static void removeWhereExpression( + Expression whereExpression, Set removeFieldNames) { if (SqlSelectHelper.isLogicExpression(whereExpression)) { BinaryExpression binaryExpression = (BinaryExpression) whereExpression; Expression leftExpression = binaryExpression.getLeftExpression(); @@ -125,7 +129,8 @@ public class SqlRemoveHelper { removeWhereExpression(leftExpression, removeFieldNames); removeWhereExpression(rightExpression, removeFieldNames); } else if (whereExpression instanceof Parenthesis) { - removeWhereExpression(((Parenthesis) whereExpression).getExpression(), removeFieldNames); + removeWhereExpression( + ((Parenthesis) whereExpression).getExpression(), removeFieldNames); } else { removeExpressionWithConstant(whereExpression, removeFieldNames); } @@ -145,22 +150,25 @@ public class SqlRemoveHelper { return constant; } - private static void removeExpressionWithConstant(Expression expression, Set removeFieldNames) { + private static void removeExpressionWithConstant( + Expression expression, Set removeFieldNames) { if (expression instanceof EqualsTo || expression instanceof GreaterThanEquals || expression instanceof GreaterThan || expression instanceof MinorThanEquals || expression instanceof MinorThan) { ComparisonOperator comparisonOperator = (ComparisonOperator) expression; - String columnName = SqlSelectHelper.getColumnName(comparisonOperator.getLeftExpression(), - comparisonOperator.getRightExpression()); + String columnName = + SqlSelectHelper.getColumnName( + comparisonOperator.getLeftExpression(), + comparisonOperator.getRightExpression()); if (!removeFieldNames.contains(columnName)) { return; } String constant = getConstant(expression); try { - ComparisonOperator constantExpression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression( - constant); + ComparisonOperator constantExpression = + (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(constant); comparisonOperator.setLeftExpression(constantExpression.getLeftExpression()); comparisonOperator.setRightExpression(constantExpression.getRightExpression()); comparisonOperator.setASTNode(constantExpression.getASTNode()); @@ -170,14 +178,16 @@ public class SqlRemoveHelper { } if (expression instanceof InExpression) { InExpression inExpression = (InExpression) expression; - String columnName = SqlSelectHelper.getColumnName(inExpression.getLeftExpression(), - inExpression.getRightExpression()); + String columnName = + SqlSelectHelper.getColumnName( + inExpression.getLeftExpression(), inExpression.getRightExpression()); if (!removeFieldNames.contains(columnName)) { return; } try { - InExpression constantExpression = (InExpression) CCJSqlParserUtil.parseCondExpression( - JsqlConstants.IN_CONSTANT); + InExpression constantExpression = + (InExpression) + CCJSqlParserUtil.parseCondExpression(JsqlConstants.IN_CONSTANT); inExpression.setLeftExpression(constantExpression.getLeftExpression()); inExpression.setRightExpression(constantExpression.getRightExpression()); inExpression.setASTNode(constantExpression.getASTNode()); @@ -187,14 +197,17 @@ public class SqlRemoveHelper { } if (expression instanceof LikeExpression) { LikeExpression likeExpression = (LikeExpression) expression; - String columnName = SqlSelectHelper.getColumnName(likeExpression.getLeftExpression(), - likeExpression.getRightExpression()); + String columnName = + SqlSelectHelper.getColumnName( + likeExpression.getLeftExpression(), + likeExpression.getRightExpression()); if (!removeFieldNames.contains(columnName)) { return; } try { - LikeExpression constantExpression = (LikeExpression) CCJSqlParserUtil.parseCondExpression( - JsqlConstants.LIKE_CONSTANT); + LikeExpression constantExpression = + (LikeExpression) + CCJSqlParserUtil.parseCondExpression(JsqlConstants.LIKE_CONSTANT); likeExpression.setLeftExpression(constantExpression.getLeftExpression()); likeExpression.setRightExpression(constantExpression.getRightExpression()); } catch (JSQLParserException e) { @@ -208,12 +221,13 @@ public class SqlRemoveHelper { if (!(selectStatement instanceof PlainSelect)) { return sql; } - selectStatement.accept(new SelectVisitorAdapter() { - @Override - public void visit(PlainSelect plainSelect) { - removeWhereCondition(plainSelect.getHaving(), removeFieldNames); - } - }); + selectStatement.accept( + new SelectVisitorAdapter() { + @Override + public void visit(PlainSelect plainSelect) { + removeWhereCondition(plainSelect.getHaving(), removeFieldNames); + } + }); return removeNumberFilter(selectStatement.toString()); } @@ -227,13 +241,16 @@ public class SqlRemoveHelper { return sql; } ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList(); - groupByExpressionList.getExpressions().removeIf(expression -> { - if (expression instanceof Column) { - Column column = (Column) expression; - return fields.contains(column.getColumnName()); - } - return false; - }); + groupByExpressionList + .getExpressions() + .removeIf( + expression -> { + if (expression instanceof Column) { + Column column = (Column) expression; + return fields.contains(column.getColumnName()); + } + return false; + }); if (CollectionUtils.isEmpty(groupByExpressionList.getExpressions())) { ((PlainSelect) selectStatement).setGroupByElement(null); } @@ -249,14 +266,15 @@ public class SqlRemoveHelper { Iterator> iterator = selectItems.iterator(); while (iterator.hasNext()) { SelectItem selectItem = iterator.next(); - selectItem.accept(new SelectItemVisitorAdapter() { - @Override - public void visit(SelectItem item) { - if (fields.contains(item.getExpression().toString())) { - iterator.remove(); - } - } - }); + selectItem.accept( + new SelectItemVisitorAdapter() { + @Override + public void visit(SelectItem item) { + if (fields.contains(item.getExpression().toString())) { + iterator.remove(); + } + } + }); } if (selectItems.isEmpty()) { selectItems.add(new SelectItem(new AllColumns())); @@ -264,15 +282,18 @@ public class SqlRemoveHelper { return selectStatement.toString(); } - public static Expression filteredExpression(Expression where, SqlEditEnum sqlEditEnum) throws Exception { + public static Expression filteredExpression(Expression where, SqlEditEnum sqlEditEnum) + throws Exception { if (Objects.isNull(where)) { return null; } if (where instanceof Parenthesis) { - Expression expression = filteredExpression(((Parenthesis) where).getExpression(), sqlEditEnum); + Expression expression = + filteredExpression(((Parenthesis) where).getExpression(), sqlEditEnum); if (expression != null) { try { - Expression parseExpression = CCJSqlParserUtil.parseExpression("(" + expression + ")"); + Expression parseExpression = + CCJSqlParserUtil.parseExpression("(" + expression + ")"); return parseExpression; } catch (JSQLParserException jsqlParserException) { log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); @@ -294,8 +315,10 @@ public class SqlRemoveHelper { private static Expression filteredLogicExpression( T binaryExpression, SqlEditEnum sqlEditEnum) throws Exception { - Expression leftExpression = filteredExpression(binaryExpression.getLeftExpression(), sqlEditEnum); - Expression rightExpression = filteredExpression(binaryExpression.getRightExpression(), sqlEditEnum); + Expression leftExpression = + filteredExpression(binaryExpression.getLeftExpression(), sqlEditEnum); + Expression rightExpression = + filteredExpression(binaryExpression.getRightExpression(), sqlEditEnum); if (leftExpression != null && rightExpression != null) { binaryExpression.setLeftExpression(leftExpression); binaryExpression.setRightExpression(rightExpression); @@ -309,13 +332,17 @@ public class SqlRemoveHelper { } } - private static Expression dealComparisonOperatorFilter(Expression expression, SqlEditEnum sqlEditEnum) { + private static Expression dealComparisonOperatorFilter( + Expression expression, SqlEditEnum sqlEditEnum) { if (Objects.isNull(expression)) { return null; } - if (expression instanceof GreaterThanEquals || expression instanceof GreaterThan - || expression instanceof MinorThan || expression instanceof MinorThanEquals - || expression instanceof EqualsTo || expression instanceof NotEqualsTo) { + if (expression instanceof GreaterThanEquals + || expression instanceof GreaterThan + || expression instanceof MinorThan + || expression instanceof MinorThanEquals + || expression instanceof EqualsTo + || expression instanceof NotEqualsTo) { return removeSingleFilter((ComparisonOperator) expression, sqlEditEnum); } else if (expression instanceof InExpression) { InExpression inExpression = (InExpression) expression; @@ -335,7 +362,8 @@ public class SqlRemoveHelper { return recursionBase(leftExpression, comparisonExpression, sqlEditEnum); } - private static Expression recursionBase(Expression leftExpression, Expression expression, SqlEditEnum sqlEditEnum) { + private static Expression recursionBase( + Expression leftExpression, Expression expression, SqlEditEnum sqlEditEnum) { if (sqlEditEnum.equals(SqlEditEnum.NUMBER_FILTER)) { return distinguishNumberFilter(leftExpression, expression); } @@ -345,7 +373,8 @@ public class SqlRemoveHelper { return expression; } - private static Expression distinguishNumberFilter(Expression leftExpression, Expression expression) { + private static Expression distinguishNumberFilter( + Expression leftExpression, Expression expression) { if (leftExpression instanceof LongValue) { return null; } else { @@ -356,6 +385,4 @@ public class SqlRemoveHelper { private static boolean isInvalidSelect(Select selectStatement) { return Objects.isNull(selectStatement) || !(selectStatement instanceof PlainSelect); } - } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java index 108156aae..8ba745e8a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java @@ -44,9 +44,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.UnaryOperator; -/** - * Sql Parser replace Helper - */ +/** Sql Parser replace Helper */ @Slf4j public class SqlReplaceHelper { @@ -55,73 +53,98 @@ public class SqlReplaceHelper { if (!(selectStatement instanceof PlainSelect)) { return sql; } - ((PlainSelect) selectStatement).getSelectItems().stream().forEach(o -> { - SelectItem selectExpressionItem = (SelectItem) o; - String alias = ""; - if (selectExpressionItem.getExpression() instanceof Function) { - Function function = (Function) selectExpressionItem.getExpression(); - Column column = (Column) function.getParameters().getExpressions().get(0); - if (fieldNameMap.containsKey(column.getColumnName())) { - String value = fieldNameMap.get(column.getColumnName()); - alias = value; - function.withParameters(new Column(value)); - } - } - if (selectExpressionItem.getExpression() instanceof Column) { - Column column = (Column) selectExpressionItem.getExpression(); - String columnName = column.getColumnName(); - if (fieldNameMap.containsKey(columnName)) { - String value = fieldNameMap.get(columnName); - alias = value; - if (StringUtils.isNotBlank(value)) { - selectExpressionItem.setExpression(new Column(value)); - } - } - } - if (Objects.nonNull(selectExpressionItem.getAlias()) && StringUtils.isNotBlank(alias)) { - selectExpressionItem.getAlias().setName(alias); - } - }); + ((PlainSelect) selectStatement) + .getSelectItems().stream() + .forEach( + o -> { + SelectItem selectExpressionItem = (SelectItem) o; + String alias = ""; + if (selectExpressionItem.getExpression() instanceof Function) { + Function function = + (Function) selectExpressionItem.getExpression(); + Column column = + (Column) + function.getParameters() + .getExpressions() + .get(0); + if (fieldNameMap.containsKey(column.getColumnName())) { + String value = fieldNameMap.get(column.getColumnName()); + alias = value; + function.withParameters(new Column(value)); + } + } + if (selectExpressionItem.getExpression() instanceof Column) { + Column column = + (Column) selectExpressionItem.getExpression(); + String columnName = column.getColumnName(); + if (fieldNameMap.containsKey(columnName)) { + String value = fieldNameMap.get(columnName); + alias = value; + if (StringUtils.isNotBlank(value)) { + selectExpressionItem.setExpression( + new Column(value)); + } + } + } + if (Objects.nonNull(selectExpressionItem.getAlias()) + && StringUtils.isNotBlank(alias)) { + selectExpressionItem.getAlias().setName(alias); + } + }); return selectStatement.toString(); } - public static String replaceAggFields(String sql, Map> fieldNameToAggMap) { + public static String replaceAggFields( + String sql, Map> fieldNameToAggMap) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; } - ((PlainSelect) selectStatement).getSelectItems().stream().forEach(o -> { - SelectItem selectExpressionItem = (SelectItem) o; - if (selectExpressionItem.getExpression() instanceof Function) { - Function function = (Function) selectExpressionItem.getExpression(); - Column column = (Column) function.getParameters().getExpressions().get(0); - if (fieldNameToAggMap.containsKey(column.getColumnName())) { - Pair agg = fieldNameToAggMap.get(column.getColumnName()); - String field = agg.getKey(); - String func = agg.getRight(); - if (AggOperatorEnum.isCountDistinct(func)) { - function.setName("count"); - function.setDistinct(true); - } else { - function.setName(func); - } - function.withParameters(new Column(field)); - if (Objects.nonNull(selectExpressionItem.getAlias()) && StringUtils.isNotBlank(field)) { - selectExpressionItem.getAlias().setName(field); - } - } - } - }); + ((PlainSelect) selectStatement) + .getSelectItems().stream() + .forEach( + o -> { + SelectItem selectExpressionItem = (SelectItem) o; + if (selectExpressionItem.getExpression() instanceof Function) { + Function function = + (Function) selectExpressionItem.getExpression(); + Column column = + (Column) + function.getParameters() + .getExpressions() + .get(0); + if (fieldNameToAggMap.containsKey(column.getColumnName())) { + Pair agg = + fieldNameToAggMap.get(column.getColumnName()); + String field = agg.getKey(); + String func = agg.getRight(); + if (AggOperatorEnum.isCountDistinct(func)) { + function.setName("count"); + function.setDistinct(true); + } else { + function.setName(func); + } + function.withParameters(new Column(field)); + if (Objects.nonNull(selectExpressionItem.getAlias()) + && StringUtils.isNotBlank(field)) { + selectExpressionItem.getAlias().setName(field); + } + } + } + }); return selectStatement.toString(); } - public static String replaceValue(String sql, Map> filedNameToValueMap) { + public static String replaceValue( + String sql, Map> filedNameToValueMap) { return replaceValue(sql, filedNameToValueMap, true); } - public static String replaceValue(String sql, Map> filedNameToValueMap, - boolean exactReplace) { + public static String replaceValue( + String sql, + Map> filedNameToValueMap, + boolean exactReplace) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; @@ -129,7 +152,8 @@ public class SqlReplaceHelper { List plainSelects = SqlSelectHelper.getPlainSelect(selectStatement); for (PlainSelect plainSelect : plainSelects) { Expression where = plainSelect.getWhere(); - FieldValueReplaceVisitor visitor = new FieldValueReplaceVisitor(exactReplace, filedNameToValueMap); + FieldValueReplaceVisitor visitor = + new FieldValueReplaceVisitor(exactReplace, filedNameToValueMap); if (Objects.nonNull(where)) { where.accept(visitor); } @@ -137,7 +161,8 @@ public class SqlReplaceHelper { return selectStatement.toString(); } - public static String replaceFieldNameByValue(String sql, Map> fieldValueToFieldNames) { + public static String replaceFieldNameByValue( + String sql, Map> fieldValueToFieldNames) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; @@ -168,11 +193,14 @@ public class SqlReplaceHelper { } else if (select instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) select; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - plainSelectList.add(subPlainSelect); - getFromSelect(subPlainSelect.getFromItem(), plainSelectList); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + getFromSelect(subPlainSelect.getFromItem(), plainSelectList); + }); } } } @@ -181,7 +209,8 @@ public class SqlReplaceHelper { return replaceFields(sql, fieldNameMap, false); } - public static String replaceFields(String sql, Map fieldNameMap, boolean exactReplace) { + public static String replaceFields( + String sql, Map fieldNameMap, boolean exactReplace) { Select selectStatement = SqlSelectHelper.getSelect(sql); List plainSelectList = SqlSelectHelper.getWithItem(selectStatement); if (selectStatement instanceof PlainSelect) { @@ -191,11 +220,14 @@ public class SqlReplaceHelper { } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - plainSelectList.add(subPlainSelect); - getFromSelect(subPlainSelect.getFromItem(), plainSelectList); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + getFromSelect(subPlainSelect.getFromItem(), plainSelectList); + }); } List orderByElements = setOperationList.getOrderByElements(); if (!CollectionUtils.isEmpty(orderByElements)) { @@ -213,16 +245,16 @@ public class SqlReplaceHelper { return selectStatement.toString(); } - private static void replaceFieldsInPlainOneSelect(Map fieldNameMap, boolean exactReplace, - PlainSelect plainSelect) { - //1. replace where fields + private static void replaceFieldsInPlainOneSelect( + Map fieldNameMap, boolean exactReplace, PlainSelect plainSelect) { + // 1. replace where fields Expression where = plainSelect.getWhere(); FieldReplaceVisitor visitor = new FieldReplaceVisitor(fieldNameMap, exactReplace); if (Objects.nonNull(where)) { where.accept(visitor); } - //2. replace select fields + // 2. replace select fields for (SelectItem selectItem : plainSelect.getSelectItems()) { selectItem.accept(visitor); replaceAsName(fieldNameMap, selectItem); @@ -237,27 +269,31 @@ public class SqlReplaceHelper { } else if (select instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) select; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + replaceFieldsInPlainOneSelect( + fieldNameMap, exactReplace, subPlainSelect); + }); } } } - //3. replace oder by fields + // 3. replace oder by fields List orderByElements = plainSelect.getOrderByElements(); if (!CollectionUtils.isEmpty(orderByElements)) { for (OrderByElement orderByElement : orderByElements) { orderByElement.accept(new OrderByReplaceVisitor(fieldNameMap, exactReplace)); } } - //4. replace group by fields + // 4. replace group by fields GroupByElement groupByElement = plainSelect.getGroupBy(); if (Objects.nonNull(groupByElement)) { groupByElement.accept(new GroupByReplaceVisitor(fieldNameMap, exactReplace)); } - //5. replace having fields + // 5. replace having fields Expression having = plainSelect.getHaving(); if (Objects.nonNull(having)) { having.accept(visitor); @@ -266,9 +302,11 @@ public class SqlReplaceHelper { if (!CollectionUtils.isEmpty(joins)) { for (Join join : joins) { if (!CollectionUtils.isEmpty(join.getOnExpressions())) { - join.getOnExpressions().stream().forEach(onExpression -> { - onExpression.accept(visitor); - }); + join.getOnExpressions().stream() + .forEach( + onExpression -> { + onExpression.accept(visitor); + }); } if (!(join.getRightItem() instanceof ParenthesedSelect)) { continue; @@ -276,7 +314,8 @@ public class SqlReplaceHelper { ParenthesedSelect parenthesedSelect = (ParenthesedSelect) join.getRightItem(); List plainSelectList = new ArrayList<>(); plainSelectList.add(parenthesedSelect.getPlainSelect()); - List subPlainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); + List subPlainSelects = + SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect subPlainSelect : subPlainSelects) { replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); } @@ -295,15 +334,14 @@ public class SqlReplaceHelper { if (StringUtils.isNotBlank(replaceFieldName)) { alias.setName(replaceFieldName); } - } public static String replaceFunction(String sql, Map functionMap) { return replaceFunction(sql, functionMap, null); } - public static String replaceFunction(String sql, Map functionMap, - Map functionCall) { + public static String replaceFunction( + String sql, Map functionMap, Map functionCall) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; @@ -317,19 +355,23 @@ public class SqlReplaceHelper { return selectStatement.toString(); } - private static void replaceFunction(Map functionMap, Map functionCall, - PlainSelect selectBody) { + private static void replaceFunction( + Map functionMap, + Map functionCall, + PlainSelect selectBody) { PlainSelect plainSelect = selectBody; - //1. replace where dataDiff function + // 1. replace where dataDiff function Expression where = plainSelect.getWhere(); - FunctionNameReplaceVisitor visitor = new FunctionNameReplaceVisitor(functionMap, functionCall); + FunctionNameReplaceVisitor visitor = + new FunctionNameReplaceVisitor(functionMap, functionCall); if (Objects.nonNull(where)) { where.accept(visitor); } GroupByElement groupBy = plainSelect.getGroupBy(); if (Objects.nonNull(groupBy)) { - GroupByFunctionReplaceVisitor replaceVisitor = new GroupByFunctionReplaceVisitor(functionMap, functionCall); + GroupByFunctionReplaceVisitor replaceVisitor = + new GroupByFunctionReplaceVisitor(functionMap, functionCall); groupBy.accept(replaceVisitor); } @@ -376,7 +418,8 @@ public class SqlReplaceHelper { } } - private static void replaceComparisonOperatorFunction(Map functionMap, Expression expression) { + private static void replaceComparisonOperatorFunction( + Map functionMap, Expression expression) { if (Objects.isNull(expression)) { return; } @@ -395,8 +438,8 @@ public class SqlReplaceHelper { } } - private static void replaceOrderByFunction(Map functionMap, - List orderByElementList) { + private static void replaceOrderByFunction( + Map functionMap, List orderByElementList) { if (Objects.isNull(orderByElementList)) { return; } @@ -429,22 +472,25 @@ public class SqlReplaceHelper { List plainSelectList = SqlSelectHelper.getWithItem(selectStatement); if (!CollectionUtils.isEmpty(plainSelectList)) { List withNameList = SqlSelectHelper.getWithName(sql); - plainSelectList.stream().forEach(plainSelect -> { - if (plainSelect.getFromItem() instanceof Table) { - Table table = (Table) plainSelect.getFromItem(); - if (!withNameList.contains(table.getName())) { - replaceSingleTable(plainSelect, tableName); - } - } - if (plainSelect.getFromItem() instanceof ParenthesedSelect) { - ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); - PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); - Table table = (Table) subPlainSelect.getFromItem(); - if (!withNameList.contains(table.getName())) { - replaceSingleTable(subPlainSelect, tableName); - } - } - }); + plainSelectList.stream() + .forEach( + plainSelect -> { + if (plainSelect.getFromItem() instanceof Table) { + Table table = (Table) plainSelect.getFromItem(); + if (!withNameList.contains(table.getName())) { + replaceSingleTable(plainSelect, tableName); + } + } + if (plainSelect.getFromItem() instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = + (ParenthesedSelect) plainSelect.getFromItem(); + PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); + Table table = (Table) subPlainSelect.getFromItem(); + if (!withNameList.contains(table.getName())) { + replaceSingleTable(subPlainSelect, tableName); + } + } + }); return selectStatement.toString(); } if (selectStatement instanceof PlainSelect) { @@ -454,11 +500,14 @@ public class SqlReplaceHelper { } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - replaceSingleTable(subPlainSelect, tableName); - replaceSubTable(subPlainSelect, tableName); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + replaceSingleTable(subPlainSelect, tableName); + replaceSubTable(subPlainSelect, tableName); + }); } } @@ -493,7 +542,9 @@ public class SqlReplaceHelper { new SelectVisitorAdapter() { @Override public void visit(PlainSelect plainSelect) { - plainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName)); + plainSelect + .getFromItem() + .accept(new TableNameReplaceVisitor(tableName)); } }); List joins = painSelect.getJoins(); @@ -502,9 +553,12 @@ public class SqlReplaceHelper { if (join.getRightItem() instanceof ParenthesedFromItem) { List plainSelectList = new ArrayList<>(); plainSelectList.add((PlainSelect) join.getRightItem()); - List subPlainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); + List subPlainSelects = + SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect subPlainSelect : subPlainSelects) { - subPlainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName)); + subPlainSelect + .getFromItem() + .accept(new TableNameReplaceVisitor(tableName)); } } else if (join.getRightItem() instanceof Table) { Table table = (Table) join.getRightItem(); @@ -532,7 +586,8 @@ public class SqlReplaceHelper { return selectStatement.toString(); } - public static String replaceHavingValue(String sql, Map> filedNameToValueMap) { + public static String replaceHavingValue( + String sql, Map> filedNameToValueMap) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; @@ -546,7 +601,8 @@ public class SqlReplaceHelper { return selectStatement.toString(); } - public static Expression distinguishDateDiffFilter(Expression leftExpression, Expression expression) { + public static Expression distinguishDateDiffFilter( + Expression leftExpression, Expression expression) { if (leftExpression instanceof Function) { Function function = (Function) leftExpression; if (function.getName().equals(JsqlConstants.DATE_FUNCTION)) { @@ -555,27 +611,35 @@ public class SqlReplaceHelper { Column field = (Column) function.getParameters().getExpressions().get(1); String columnName = field.getColumnName(); try { - String startDateValue = DateFunctionHelper.getStartDateStr(comparisonOperator, leftExpressions); + String startDateValue = + DateFunctionHelper.getStartDateStr(comparisonOperator, leftExpressions); String endDateValue = DateFunctionHelper.getEndDateValue(leftExpressions); String dateOperator = comparisonOperator.getStringExpression(); String endDateOperator = JsqlConstants.rightMap.get(dateOperator); String startDateOperator = JsqlConstants.leftMap.get(dateOperator); - String endDateCondExpr = columnName + endDateOperator + StringUtil.getCommaWrap(endDateValue); - ComparisonOperator rightExpression = (ComparisonOperator) - CCJSqlParserUtil.parseCondExpression(endDateCondExpr); + String endDateCondExpr = + columnName + endDateOperator + StringUtil.getCommaWrap(endDateValue); + ComparisonOperator rightExpression = + (ComparisonOperator) + CCJSqlParserUtil.parseCondExpression(endDateCondExpr); - String startDateCondExpr = columnName + StringUtil.getSpaceWrap(startDateOperator) - + StringUtil.getCommaWrap(startDateValue); - ComparisonOperator newLeftExpression = (ComparisonOperator) - CCJSqlParserUtil.parseCondExpression(startDateCondExpr); + String startDateCondExpr = + columnName + + StringUtil.getSpaceWrap(startDateOperator) + + StringUtil.getCommaWrap(startDateValue); + ComparisonOperator newLeftExpression = + (ComparisonOperator) + CCJSqlParserUtil.parseCondExpression(startDateCondExpr); - AndExpression andExpression = new AndExpression(newLeftExpression, rightExpression); + AndExpression andExpression = + new AndExpression(newLeftExpression, rightExpression); if (JsqlConstants.GREATER_THAN.equals(dateOperator) || JsqlConstants.GREATER_THAN_EQUALS.equals(dateOperator)) { return newLeftExpression; } else { - return CCJSqlParserUtil.parseCondExpression("(" + andExpression.toString() + ")"); + return CCJSqlParserUtil.parseCondExpression( + "(" + andExpression.toString() + ")"); } } catch (JSQLParserException e) { log.error("JSQLParserException", e); @@ -597,7 +661,8 @@ public class SqlReplaceHelper { if (Objects.nonNull(f.getAlias()) && f.getExpression() instanceof Function) { Function function = (Function) f.getExpression(); String alias = f.getAlias().getName(); - if (function.getParameters().size() == 1 && function.getParameters().get(0) instanceof Column) { + if (function.getParameters().size() == 1 + && function.getParameters().get(0) instanceof Column) { Column column = (Column) function.getParameters().get(0); if (column.getColumnName().equalsIgnoreCase(alias)) { selectNames.put(alias, String.valueOf(i + 1)); @@ -605,21 +670,30 @@ public class SqlReplaceHelper { } } } - plainSelect.getOrderByElements().stream().forEach(o -> { - if (o.getExpression() instanceof Function) { - Function function = (Function) o.getExpression(); - if (function.getParameters().size() == 1 && function.getParameters().get(0) instanceof Column) { - Column column = (Column) function.getParameters().get(0); - if (selectNames.containsKey(column.getColumnName())) { - o.setExpression(new LongValue(selectNames.get(column.getColumnName()))); - } - } - } - }); + plainSelect.getOrderByElements().stream() + .forEach( + o -> { + if (o.getExpression() instanceof Function) { + Function function = (Function) o.getExpression(); + if (function.getParameters().size() == 1 + && function.getParameters().get(0) + instanceof Column) { + Column column = + (Column) function.getParameters().get(0); + if (selectNames.containsKey(column.getColumnName())) { + o.setExpression( + new LongValue( + selectNames.get( + column.getColumnName()))); + } + } + } + }); } if (plainSelect.getFromItem() instanceof ParenthesedSelect) { ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); - parenthesedSelect.setSelect(replaceAggAliasOrderItem(parenthesedSelect.getSelect())); + parenthesedSelect.setSelect( + replaceAggAliasOrderItem(parenthesedSelect.getSelect())); } return selectStatement; } @@ -637,7 +711,8 @@ public class SqlReplaceHelper { if (expression instanceof Column && replace.containsKey(expr)) { return replace.get(expr); } - ExpressionReplaceVisitor expressionReplaceVisitor = new ExpressionReplaceVisitor(replace); + ExpressionReplaceVisitor expressionReplaceVisitor = + new ExpressionReplaceVisitor(replace); expression.accept(expressionReplaceVisitor); return expression.toString(); } @@ -652,10 +727,13 @@ public class SqlReplaceHelper { } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - plainSelectList.add(subPlainSelect); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + }); } } else { return sql; @@ -667,8 +745,10 @@ public class SqlReplaceHelper { return selectStatement.toString(); } - private static void replacePlainSelectByExpr(PlainSelect plainSelect, Map replace) { - QueryExpressionReplaceVisitor expressionReplaceVisitor = new QueryExpressionReplaceVisitor(replace); + private static void replacePlainSelectByExpr( + PlainSelect plainSelect, Map replace) { + QueryExpressionReplaceVisitor expressionReplaceVisitor = + new QueryExpressionReplaceVisitor(replace); for (SelectItem selectItem : plainSelect.getSelectItems()) { selectItem.accept(expressionReplaceVisitor); } @@ -686,7 +766,8 @@ public class SqlReplaceHelper { if (!CollectionUtils.isEmpty(orderByElements)) { for (OrderByElement orderByElement : orderByElements) { orderByElement.setExpression( - QueryExpressionReplaceVisitor.replace(orderByElement.getExpression(), replace)); + QueryExpressionReplaceVisitor.replace( + orderByElement.getExpression(), replace)); } } } @@ -694,19 +775,22 @@ public class SqlReplaceHelper { public static String dealAliasToOrderBy(String querySql) { Select selectStatement = SqlSelectHelper.getSelect(querySql); List plainSelectList = new ArrayList<>(); - //List withPlainSelectList = SqlSelectHelper.getWithItem(selectStatement); - //if (!CollectionUtils.isEmpty(withPlainSelectList)) { + // List withPlainSelectList = SqlSelectHelper.getWithItem(selectStatement); + // if (!CollectionUtils.isEmpty(withPlainSelectList)) { // plainSelectList.addAll(withPlainSelectList); - //} + // } if (selectStatement instanceof PlainSelect) { plainSelectList.add((PlainSelect) selectStatement); } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - plainSelectList.add(subPlainSelect); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + }); } } for (PlainSelect plainSelect : plainSelectList) { @@ -718,19 +802,20 @@ public class SqlReplaceHelper { Map map = new HashMap<>(); for (int i = 0; i < selectItemList.size(); i++) { if (!Objects.isNull(selectItemList.get(i).getAlias())) { - map.put(selectItemList.get(i).getAlias().getName(), selectItemList.get(i).getExpression()); + map.put( + selectItemList.get(i).getAlias().getName(), + selectItemList.get(i).getExpression()); selectItemList.get(i).setAlias(null); } } for (OrderByElement orderByElement : orderByElementList) { if (map.containsKey(orderByElement.getExpression().toString())) { - orderByElement.setExpression(map.get(orderByElement.getExpression().toString())); + orderByElement.setExpression( + map.get(orderByElement.getExpression().toString())); } } plainSelect.setOrderByElements(orderByElementList); } return selectStatement.toString(); } - } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelper.java index fcc9b4a3d..019063ab5 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelper.java @@ -1,14 +1,6 @@ package com.tencent.supersonic.common.jsqlparser; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; @@ -21,13 +13,21 @@ import net.sf.jsqlparser.statement.select.SelectItem; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; -/** - * Sql Parser Select function Helper - */ +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** Sql Parser Select function Helper */ @Slf4j public class SqlSelectFunctionHelper { - public static List aggregateFunctionName = Arrays.asList("SUM", "COUNT", "MAX", "MIN", "AVG"); + public static List aggregateFunctionName = + Arrays.asList("SUM", "COUNT", "MAX", "MIN", "AVG"); public static boolean hasAggregateFunction(String sql) { if (!CollectionUtils.isEmpty(getFunctions(sql))) { @@ -58,7 +58,8 @@ public class SqlSelectFunctionHelper { return visitor.getFunctionNames(); } - public static Function getFunction(Expression expression, Map fieldNameToAggregate) { + public static Function getFunction( + Expression expression, Map fieldNameToAggregate) { if (!(expression instanceof Column)) { return null; } @@ -100,7 +101,8 @@ public class SqlSelectFunctionHelper { expression.accept(visitor); Set functions = visitor.getFunctionNames(); return functions.stream() - .filter(t -> aggregateFunctionName.contains(t.toUpperCase())).collect(Collectors.toList()); + .filter(t -> aggregateFunctionName.contains(t.toUpperCase())) + .collect(Collectors.toList()); } return new ArrayList<>(); } @@ -119,4 +121,3 @@ public class SqlSelectFunctionHelper { return false; } } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java index 60e743975..410180ae8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java @@ -51,9 +51,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/** - * Sql Parser Select Helper - */ +/** Sql Parser Select Helper */ @Slf4j public class SqlSelectHelper { @@ -73,9 +71,12 @@ public class SqlSelectHelper { having.accept(new FieldAndValueAcquireVisitor(result)); } } - result = result.stream() - .filter(fieldExpression -> StringUtils.isNotBlank(fieldExpression.getFieldName())) - .collect(Collectors.toSet()); + result = + result.stream() + .filter( + fieldExpression -> + StringUtils.isNotBlank(fieldExpression.getFieldName())) + .collect(Collectors.toSet()); return new ArrayList<>(result); } @@ -90,27 +91,31 @@ public class SqlSelectHelper { } public static void getWhereFields(List plainSelectList, Set result) { - plainSelectList.stream().forEach(plainSelect -> { - Expression where = plainSelect.getWhere(); - if (Objects.nonNull(where)) { - where.accept(new FieldAcquireVisitor(result)); - } - }); + plainSelectList.stream() + .forEach( + plainSelect -> { + Expression where = plainSelect.getWhere(); + if (Objects.nonNull(where)) { + where.accept(new FieldAcquireVisitor(result)); + } + }); } public static List gePureSelectFields(String sql) { List plainSelectList = getPlainSelect(sql); Set result = new HashSet<>(); - plainSelectList.stream().forEach(plainSelect -> { - List> selectItems = plainSelect.getSelectItems(); - for (SelectItem selectItem : selectItems) { - if (!(selectItem.getExpression() instanceof Column)) { - continue; - } - Column column = (Column) selectItem.getExpression(); - result.add(column.getColumnName()); - } - }); + plainSelectList.stream() + .forEach( + plainSelect -> { + List> selectItems = plainSelect.getSelectItems(); + for (SelectItem selectItem : selectItems) { + if (!(selectItem.getExpression() instanceof Column)) { + continue; + } + Column column = (Column) selectItem.getExpression(); + result.add(column.getColumnName()); + } + }); return new ArrayList<>(result); } @@ -124,12 +129,14 @@ public class SqlSelectHelper { public static Set getSelectFields(List plainSelectList) { Set result = new HashSet<>(); - plainSelectList.stream().forEach(plainSelect -> { - List> selectItems = plainSelect.getSelectItems(); - for (SelectItem selectItem : selectItems) { - selectItem.accept(new FieldAcquireVisitor(result)); - } - }); + plainSelectList.stream() + .forEach( + plainSelect -> { + List> selectItems = plainSelect.getSelectItems(); + for (SelectItem selectItem : selectItems) { + selectItem.accept(new FieldAcquireVisitor(result)); + } + }); return result; } @@ -146,10 +153,13 @@ public class SqlSelectHelper { } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - getSubPlainSelect(subPlainSelect, plainSelectList); - }); + setOperationList + .getSelects() + .forEach( + subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + getSubPlainSelect(subPlainSelect, plainSelectList); + }); } } return plainSelectList; @@ -226,37 +236,39 @@ public class SqlSelectHelper { List plainSelects = new ArrayList<>(); for (PlainSelect plainSelect : plainSelectList) { plainSelects.add(plainSelect); - ExpressionVisitorAdapter expressionVisitor = new ExpressionVisitorAdapter() { - @Override - public void visit(Select subSelect) { - if (subSelect instanceof ParenthesedSelect) { - ParenthesedSelect parenthesedSelect = (ParenthesedSelect) subSelect; - if (parenthesedSelect.getSelect() instanceof PlainSelect) { - plainSelects.add(parenthesedSelect.getPlainSelect()); + ExpressionVisitorAdapter expressionVisitor = + new ExpressionVisitorAdapter() { + @Override + public void visit(Select subSelect) { + if (subSelect instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) subSelect; + if (parenthesedSelect.getSelect() instanceof PlainSelect) { + plainSelects.add(parenthesedSelect.getPlainSelect()); + } + } } - } - } - }; + }; - plainSelect.accept(new SelectVisitorAdapter() { - @Override - public void visit(PlainSelect plainSelect) { - Expression whereExpression = plainSelect.getWhere(); - if (whereExpression != null) { - whereExpression.accept(expressionVisitor); - } - Expression having = plainSelect.getHaving(); - if (Objects.nonNull(having)) { - having.accept(expressionVisitor); - } - List> selectItems = plainSelect.getSelectItems(); - if (!CollectionUtils.isEmpty(selectItems)) { - for (SelectItem selectItem : selectItems) { - selectItem.accept(expressionVisitor); + plainSelect.accept( + new SelectVisitorAdapter() { + @Override + public void visit(PlainSelect plainSelect) { + Expression whereExpression = plainSelect.getWhere(); + if (whereExpression != null) { + whereExpression.accept(expressionVisitor); + } + Expression having = plainSelect.getHaving(); + if (Objects.nonNull(having)) { + having.accept(expressionVisitor); + } + List> selectItems = plainSelect.getSelectItems(); + if (!CollectionUtils.isEmpty(selectItems)) { + for (SelectItem selectItem : selectItems) { + selectItem.accept(expressionVisitor); + } + } } - } - } - }); + }); } return plainSelects; } @@ -297,17 +309,19 @@ public class SqlSelectHelper { if (Objects.nonNull(having)) { having.accept(new FieldAcquireVisitor(result)); } - } private static void getLateralViewsFields(PlainSelect plainSelect, Set result) { List lateralViews = plainSelect.getLateralViews(); if (!CollectionUtils.isEmpty(lateralViews)) { - lateralViews.stream().forEach(l -> { - if (Objects.nonNull(l.getGeneratorFunction())) { - l.getGeneratorFunction().accept(new FieldAcquireVisitor(result)); - } - }); + lateralViews.stream() + .forEach( + l -> { + if (Objects.nonNull(l.getGeneratorFunction())) { + l.getGeneratorFunction() + .accept(new FieldAcquireVisitor(result)); + } + }); } } @@ -412,10 +426,11 @@ public class SqlSelectHelper { private static void getOrderByFields(PlainSelect plainSelect, Set result) { Set orderByFieldExpressions = getOrderByFields(plainSelect); - Set collect = orderByFieldExpressions.stream() - .map(fieldExpression -> fieldExpression.getFieldName()) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); + Set collect = + orderByFieldExpressions.stream() + .map(fieldExpression -> fieldExpression.getFieldName()) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); result.addAll(collect); } @@ -474,12 +489,13 @@ public class SqlSelectHelper { if (selectItem.getExpression() instanceof Function) { Function function = (Function) selectItem.getExpression(); if (Objects.nonNull(function.getParameters()) - && !CollectionUtils.isEmpty(function.getParameters().getExpressions())) { - String columnName = function.getParameters().getExpressions().get(0).toString(); + && !CollectionUtils.isEmpty( + function.getParameters().getExpressions())) { + String columnName = + function.getParameters().getExpressions().get(0).toString(); result.add(columnName); } } - } } return new ArrayList<>(result); @@ -502,13 +518,14 @@ public class SqlSelectHelper { result.add(alias.getName()); } else { if (Objects.nonNull(function.getParameters()) - && !CollectionUtils.isEmpty(function.getParameters().getExpressions())) { - String columnName = function.getParameters().getExpressions().get(0).toString(); + && !CollectionUtils.isEmpty( + function.getParameters().getExpressions())) { + String columnName = + function.getParameters().getExpressions().get(0).toString(); result.add(columnName); } } } - } } return new ArrayList<>(result); @@ -542,8 +559,9 @@ public class SqlSelectHelper { } public static boolean isLogicExpression(Expression whereExpression) { - return whereExpression instanceof AndExpression || (whereExpression instanceof OrExpression - || (whereExpression instanceof XorExpression)); + return whereExpression instanceof AndExpression + || (whereExpression instanceof OrExpression + || (whereExpression instanceof XorExpression)); } public static String getColumnName(Expression leftExpression, Expression rightExpression) { @@ -553,7 +571,8 @@ public class SqlSelectHelper { if (leftExpression instanceof Function) { ExpressionList expressionList = ((Function) leftExpression).getParameters(); - if (!CollectionUtils.isEmpty(expressionList) && expressionList.get(0) instanceof Column) { + if (!CollectionUtils.isEmpty(expressionList) + && expressionList.get(0) instanceof Column) { return ((Column) expressionList.get(0)).getColumnName(); } } @@ -619,7 +638,8 @@ public class SqlSelectHelper { PlainSelect withPlainSelect = (PlainSelect) withSelect; plainSelectList.add(withPlainSelect); if (withPlainSelect.getFromItem() instanceof ParenthesedSelect) { - ParenthesedSelect parenthesedSelect = (ParenthesedSelect) withPlainSelect.getFromItem(); + ParenthesedSelect parenthesedSelect = + (ParenthesedSelect) withPlainSelect.getFromItem(); plainSelectList.add(parenthesedSelect.getPlainSelect()); } } @@ -666,7 +686,8 @@ public class SqlSelectHelper { } if (plainSelect.getFromItem() instanceof ParenthesedSelect) { - PlainSelect subSelect = ((ParenthesedSelect) plainSelect.getFromItem()).getPlainSelect(); + PlainSelect subSelect = + ((ParenthesedSelect) plainSelect.getFromItem()).getPlainSelect(); return getTable(subSelect.getSelectBody().toString()); } @@ -775,7 +796,8 @@ public class SqlSelectHelper { return results; } - private static void getFieldsWithSubQuery(PlainSelect plainSelect, Map> fields) { + private static void getFieldsWithSubQuery( + PlainSelect plainSelect, Map> fields) { if (plainSelect.getFromItem() instanceof Table) { List withAlias = new ArrayList<>(); if (!CollectionUtils.isEmpty(plainSelect.getWithItemsList())) { @@ -792,9 +814,10 @@ public class SqlSelectHelper { if (!fields.containsKey(table.getFullyQualifiedName())) { fields.put(tableName, new HashSet<>()); } - List sqlFields = getFieldsByPlainSelect(plainSelect).stream().map(f -> f.replaceAll("`", "")) - .collect( - Collectors.toList()); + List sqlFields = + getFieldsByPlainSelect(plainSelect).stream() + .map(f -> f.replaceAll("`", "")) + .collect(Collectors.toList()); fields.get(tableName).addAll(sqlFields); } } @@ -806,13 +829,14 @@ public class SqlSelectHelper { } for (Join join : plainSelect.getJoins()) { if (join.getRightItem() instanceof ParenthesedSelect) { - getFieldsWithSubQuery(((ParenthesedSelect) join.getRightItem()).getPlainSelect(), fields); + getFieldsWithSubQuery( + ((ParenthesedSelect) join.getRightItem()).getPlainSelect(), fields); } if (join.getFromItem() instanceof ParenthesedSelect) { - getFieldsWithSubQuery(((ParenthesedSelect) join.getFromItem()).getPlainSelect(), fields); + getFieldsWithSubQuery( + ((ParenthesedSelect) join.getFromItem()).getPlainSelect(), fields); } } } } } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelper.java index e378cc580..638fec08f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelper.java @@ -1,13 +1,12 @@ package com.tencent.supersonic.common.jsqlparser; -import java.util.List; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.collections.CollectionUtils; -/** - * Sql Parser valid Helper - */ +import java.util.List; + +/** Sql Parser valid Helper */ @Slf4j public class SqlValidHelper { @@ -19,7 +18,7 @@ public class SqlValidHelper { * @return */ public static boolean equals(String thisSql, String otherSql) { - //1. select fields + // 1. select fields List thisSelectFields = SqlSelectHelper.getSelectFields(thisSql); List otherSelectFields = SqlSelectHelper.getSelectFields(otherSql); @@ -27,7 +26,7 @@ public class SqlValidHelper { return false; } - //2. all fields + // 2. all fields List thisAllFields = SqlSelectHelper.getAllSelectFields(thisSql); List otherAllFields = SqlSelectHelper.getAllSelectFields(otherSql); @@ -35,28 +34,31 @@ public class SqlValidHelper { return false; } - //3. where + // 3. where List thisFieldExpressions = SqlSelectHelper.getFilterExpression(thisSql); List otherFieldExpressions = SqlSelectHelper.getFilterExpression(otherSql); if (!CollectionUtils.isEqualCollection(thisFieldExpressions, otherFieldExpressions)) { return false; } - //4. tableName + // 4. tableName if (!SqlSelectHelper.getDbTableName(thisSql) .equalsIgnoreCase(SqlSelectHelper.getDbTableName(otherSql))) { return false; } - //5. having + // 5. having List thisHavingExpressions = SqlSelectHelper.getHavingExpressions(thisSql); - List otherHavingExpressions = SqlSelectHelper.getHavingExpressions(otherSql); + List otherHavingExpressions = + SqlSelectHelper.getHavingExpressions(otherSql); if (!CollectionUtils.isEqualCollection(thisHavingExpressions, otherHavingExpressions)) { return false; } - //6. orderBy - List thisOrderByExpressions = SqlSelectHelper.getOrderByExpressions(thisSql); - List otherOrderByExpressions = SqlSelectHelper.getOrderByExpressions(otherSql); + // 6. orderBy + List thisOrderByExpressions = + SqlSelectHelper.getOrderByExpressions(thisSql); + List otherOrderByExpressions = + SqlSelectHelper.getOrderByExpressions(otherSql); if (!CollectionUtils.isEqualCollection(thisOrderByExpressions, otherOrderByExpressions)) { return false; @@ -73,6 +75,4 @@ public class SqlValidHelper { return false; } } - } - diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java index c8f0d2389..89a67bfd8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java @@ -15,4 +15,4 @@ public class TableNameReplaceVisitor extends FromItemVisitorAdapter { public void visit(Table table) { table.setName(tableName); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java b/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java index 77a11d18d..c8662b8bb 100644 --- a/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java +++ b/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java @@ -14,5 +14,4 @@ public class SystemConfigDO { private String parameters; private String admin; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java b/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java index 982bc5a8e..bcc0e07db 100644 --- a/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java +++ b/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java @@ -5,7 +5,4 @@ import com.tencent.supersonic.common.persistence.dataobject.SystemConfigDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface SystemConfigMapper extends BaseMapper { - - -} \ No newline at end of file +public interface SystemConfigMapper extends BaseMapper {} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java index 5e798ecb9..c877ee515 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.common.pojo; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; - -import java.util.List; import javax.validation.constraints.NotBlank; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import lombok.Data; +import java.util.List; + @Data public class Aggregator { @@ -21,8 +21,7 @@ public class Aggregator { private String alias; - public Aggregator() { - } + public Aggregator() {} public Aggregator(String column, AggOperatorEnum func) { this.column = column; @@ -44,16 +43,11 @@ public class Aggregator { @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"column\":\"") - .append(column).append('\"'); - sb.append(",\"func\":") - .append(func); - sb.append(",\"nameCh\":\"") - .append(nameCh).append('\"'); - sb.append(",\"args\":") - .append(args); - sb.append(",\"alias\":") - .append(alias); + sb.append("\"column\":\"").append(column).append('\"'); + sb.append(",\"func\":").append(func); + sb.append(",\"nameCh\":\"").append(nameCh).append('\"'); + sb.append(",\"args\":").append(args); + sb.append(",\"alias\":").append(alias); sb.append('}'); return sb.toString(); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ChatModelConfig.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ChatModelConfig.java index 6bacb3291..56a2bf4a1 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ChatModelConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ChatModelConfig.java @@ -31,4 +31,4 @@ public class ChatModelConfig implements Serializable { public String keyDecrypt() { return AESEncryptionUtil.aesDecryptECB(getApiKey()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ColumnOrder.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ColumnOrder.java index 69311ecb8..12afd344d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ColumnOrder.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ColumnOrder.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.common.pojo; -import java.io.Serializable; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.io.Serializable; + @Data @AllArgsConstructor @NoArgsConstructor diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java index 7e34bc6e8..9f551110d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.common.pojo; import java.util.regex.Pattern; - public class Constants { public static final String COMMA = ","; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Criterion.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Criterion.java index 1b9e40ef9..4240b4079 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Criterion.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Criterion.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.common.pojo; - import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import lombok.Data; + import java.util.Arrays; import java.util.List; -import lombok.Data; @Data public class Criterion { @@ -26,7 +26,8 @@ public class Criterion { this.value = value; this.dataType = dataType; - if (FilterOperatorEnum.BETWEEN.name().equals(operator) || FilterOperatorEnum.IN.name().equals(operator) + if (FilterOperatorEnum.BETWEEN.name().equals(operator) + || FilterOperatorEnum.IN.name().equals(operator) || FilterOperatorEnum.NOT_IN.name().equals(operator)) { this.values = (List) value; } @@ -34,11 +35,11 @@ public class Criterion { public boolean isNeedApostrophe() { return Arrays.stream(StringDataType.values()) - .filter(value -> this.dataType.equalsIgnoreCase(value.getType())).findFirst() + .filter(value -> this.dataType.equalsIgnoreCase(value.getType())) + .findFirst() .isPresent(); } - public enum NumericDataType { TINYINT("TINYINT"), SMALLINT("SMALLINT"), @@ -62,7 +63,6 @@ public class Criterion { } } - public enum StringDataType { VARCHAR("VARCHAR"), STRING("STRING"), @@ -77,6 +77,4 @@ public class Criterion { return type; } } - - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java index 92a865f07..87a8d0a1d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java @@ -1,10 +1,9 @@ package com.tencent.supersonic.common.pojo; - import com.tencent.supersonic.common.pojo.enums.EventType; import org.springframework.context.ApplicationEvent; -import java.util.List; +import java.util.List; public class DataEvent extends ApplicationEvent { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataFormat.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataFormat.java index b4e23ce71..38bf6f95a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataFormat.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataFormat.java @@ -8,6 +8,4 @@ public class DataFormat { private boolean needMultiply100; private Integer decimalPlaces; - - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java index 5974dbb3a..3ec65c8b2 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java @@ -8,9 +8,7 @@ import lombok.Data; @Builder public class DataItem { - /*** - * This field uses an underscore (_) at the end. - */ + /** * This field uses an underscore (_) at the end. */ private String id; private String bizName; @@ -21,9 +19,7 @@ public class DataItem { private TypeEnums type; - /*** - * This field uses an underscore (_) at the end. - */ + /** * This field uses an underscore (_) at the end. */ private String modelId; private String defaultAgg; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java index 74c61e390..0fe9c28de 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java @@ -11,7 +11,8 @@ public class DataUpdateEvent extends ApplicationEvent { private Long id; private TypeEnums type; - public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, TypeEnums type) { + public DataUpdateEvent( + Object source, String name, String newName, Long modelId, Long id, TypeEnums type) { super(source); this.name = name; this.newName = newName; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java index 17cc06115..8f4d9e52f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java @@ -8,6 +8,7 @@ import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; import java.util.Objects; + import static java.time.LocalDate.now; @Data @@ -15,34 +16,23 @@ public class DateConf { private static final long serialVersionUID = 3074129990945004340L; - private DateMode dateMode = DateMode.RECENT; - /** - * like 2021-10-22, dateMode=1 - */ + /** like 2021-10-22, dateMode=1 */ private String startDate = now().plusDays(-1).toString(); + private String endDate = now().toString(); - /** - * [2021-10-22, 2022-01-22], dateMode=2 - */ + /** [2021-10-22, 2022-01-22], dateMode=2 */ private List dateList = new ArrayList<>(); - /** - * the last unit time unit, - * such as the last 7 days, unit = 7 - */ + /** the last unit time unit, such as the last 7 days, unit = 7 */ private Integer unit = 1; - /** - * DAY,WEEK,MONTH - */ + /** DAY,WEEK,MONTH */ private String period = Constants.DAY; - /** - * the text parse from , example "last 7 days" , "last mouth" - */ + /** the text parse from , example "last 7 days" , "last mouth" */ private String detectWord; private boolean isInherited; @@ -93,33 +83,28 @@ public class DateConf { public enum DateMode { /** - * date mode - * 1 - BETWEEN, continuous static value, [startDate, endDate] - * 2 - LIST, discrete static value, [dateList] - * 3 - RECENT, dynamic time related to the actual available time of the element, [unit, period] - * 4 - AVAILABLE, dynamic time which guaranteed to query some data, [startDate, endDate] - * 5 - ALL, all table data + * date mode 1 - BETWEEN, continuous static value, [startDate, endDate] 2 - LIST, discrete + * static value, [dateList] 3 - RECENT, dynamic time related to the actual available time of + * the element, [unit, period] 4 - AVAILABLE, dynamic time which guaranteed to query some + * data, [startDate, endDate] 5 - ALL, all table data */ - BETWEEN, LIST, RECENT, AVAILABLE, ALL + BETWEEN, + LIST, + RECENT, + AVAILABLE, + ALL } @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"dateMode\":") - .append(dateMode); - sb.append(",\"startDate\":\"") - .append(startDate).append('\"'); - sb.append(",\"endDate\":\"") - .append(endDate).append('\"'); - sb.append(",\"dateList\":") - .append(dateList); - sb.append(",\"unit\":") - .append(unit); - sb.append(",\"period\":\"") - .append(period).append('\"'); - sb.append(",\"text\":\"") - .append(detectWord).append('\"'); + sb.append("\"dateMode\":").append(dateMode); + sb.append(",\"startDate\":\"").append(startDate).append('\"'); + sb.append(",\"endDate\":\"").append(endDate).append('\"'); + sb.append(",\"dateList\":").append(dateList); + sb.append(",\"unit\":").append(unit); + sb.append(",\"period\":\"").append(period).append('\"'); + sb.append(",\"text\":\"").append(detectWord).append('\"'); sb.append('}'); return sb.toString(); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java index 238309d70..7ea5ecbdf 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java @@ -4,7 +4,5 @@ public class DimensionConstants { public static final String DIMENSION_TIME_FORMAT = "time_format"; - public static final String DIMENSION_TYPE = "dimension_type"; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingModelConfig.java b/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingModelConfig.java index 17427ed68..c3926ee77 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingModelConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingModelConfig.java @@ -24,4 +24,4 @@ public class EmbeddingModelConfig implements Serializable { private Integer maxToken; private Boolean logRequests; private Boolean logResponses; -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingStoreConfig.java b/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingStoreConfig.java index aa6dfff5d..e3703f6f8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingStoreConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/EmbeddingStoreConfig.java @@ -22,4 +22,4 @@ public class EmbeddingStoreConfig implements Serializable { private Long timeOut = 60L; private Integer dimension; private String databaseName; -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Filter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Filter.java index c3037ce3a..6f79368f1 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Filter.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Filter.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.common.pojo; - import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; + import java.util.List; @Data @@ -36,23 +36,19 @@ public class Filter { @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"relation\":") - .append(relation); - sb.append(",\"bizName\":\"") - .append(bizName).append('\"'); - sb.append(",\"name\":\"") - .append(name).append('\"'); - sb.append(",\"operator\":") - .append(operator); - sb.append(",\"value\":") - .append(value); - sb.append(",\"children\":") - .append(children); + sb.append("\"relation\":").append(relation); + sb.append(",\"bizName\":\"").append(bizName).append('\"'); + sb.append(",\"name\":\"").append(name).append('\"'); + sb.append(",\"operator\":").append(operator); + sb.append(",\"value\":").append(value); + sb.append(",\"children\":").append(children); sb.append('}'); return sb.toString(); } public enum Relation { - FILTER, OR, AND + FILTER, + OR, + AND } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ItemDateResp.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ItemDateResp.java index 4cd941a0c..e2564addc 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ItemDateResp.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ItemDateResp.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.common.pojo; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.ToString; +import java.util.List; + @Data @AllArgsConstructor @Builder @@ -17,4 +18,4 @@ public class ItemDateResp { private String endDate; private String datePeriod; private List unavailableDateList; -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java b/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java index e90c1d2ff..3cd430e92 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java @@ -15,5 +15,4 @@ public class JoinCondition { private String rightField; private FilterOperatorEnum operator; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelConfig.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelConfig.java index ba5dddacc..c7b405867 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelConfig.java @@ -13,5 +13,4 @@ public class ModelConfig implements Serializable { private static final long serialVersionUID = 1L; private ChatModelConfig chatModel; private EmbeddingModelConfig embeddingModel; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java index f1f2d415b..451db3e01 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.pojo; - import lombok.Data; import java.util.List; @@ -16,9 +15,8 @@ public class ModelRela extends RecordInfo { private Long toModelId; - //left join, inner join, right join, outer join + // left join, inner join, right join, outer join private String joinType; private List joinConditions; - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java index 9e8b70b36..198a7faac 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.common.pojo; -import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER; - -import com.google.common.base.Objects; import javax.validation.constraints.NotBlank; +import com.google.common.base.Objects; import lombok.Data; +import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER; + @Data public class Order { @@ -20,16 +20,13 @@ public class Order { this.direction = direction; } - public Order() { - } + public Order() {} @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"column\":\"") - .append(column).append('\"'); - sb.append(",\"direction\":\"") - .append(direction).append('\"'); + sb.append("\"column\":\"").append(column).append('\"'); + sb.append(",\"direction\":\"").append(direction).append('\"'); sb.append('}'); return sb.toString(); } @@ -43,8 +40,7 @@ public class Order { return false; } Order order = (Order) o; - return Objects.equal(column, order.column) && Objects.equal(direction, - order.direction); + return Objects.equal(column, order.column) && Objects.equal(direction, order.direction); } @Override diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/PageBaseReq.java b/common/src/main/java/com/tencent/supersonic/common/pojo/PageBaseReq.java index 68257ea53..2af7e5d94 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/PageBaseReq.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/PageBaseReq.java @@ -18,5 +18,4 @@ public class PageBaseReq { public Integer getLimitSize() { return this.pageSize; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java index e38a103d7..67c4e6323 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java @@ -13,42 +13,24 @@ import java.util.Map; /** * 1.Password Field: * - * dataType: string - * name: password - * require: true/false or any value/empty - * placeholder: 'Please enter the relevant configuration information' - * value: initial value - * Text Input Field: + *

dataType: string name: password require: true/false or any value/empty placeholder: 'Please + * enter the relevant configuration information' value: initial value Text Input Field: * - * 2.dataType: string - * require: true/false or any value/empty - * placeholder: 'Please enter the relevant configuration information' - * value: initial value - * Long Text Input Field: + *

2.dataType: string require: true/false or any value/empty placeholder: 'Please enter the + * relevant configuration information' value: initial value Long Text Input Field: * - * 3.dataType: longText - * require: true/false or any value/empty - * placeholder: 'Please enter the relevant configuration information' - * value: initial value - * Number Input Field: + *

3.dataType: longText require: true/false or any value/empty placeholder: 'Please enter the + * relevant configuration information' value: initial value Number Input Field: * - * 4.dataType: number - * require: true/false or any value/empty - * placeholder: 'Please enter the relevant configuration information' - * value: initial value - * Switch Component: + *

4.dataType: number require: true/false or any value/empty placeholder: 'Please enter the + * relevant configuration information' value: initial value Switch Component: * - * 5.dataType: bool - * require: true/false or any value/empty - * value: initial value - * Select Dropdown Component: + *

5.dataType: bool require: true/false or any value/empty value: initial value Select Dropdown + * Component: * - * 6.dataType: list - * candidateValues: ["OPEN_AI", "OLLAMA"] or - * [{label: 'Model Name 1', value: 'OPEN_AI'}, {label: 'Model Name 2', value: 'OLLAMA'}] - * require: true/false or any value/empty - * placeholder: 'Please enter the relevant configuration information' - * value: initial value + *

6.dataType: list candidateValues: ["OPEN_AI", "OLLAMA"] or [{label: 'Model Name 1', value: + * 'OPEN_AI'}, {label: 'Model Name 2', value: 'OLLAMA'}] require: true/false or any value/empty + * placeholder: 'Please enter the relevant configuration information' value: initial value */ public class Parameter { private String name; @@ -61,18 +43,36 @@ public class Parameter { private List candidateValues; private List dependencies; - public Parameter(String name, String defaultValue, String comment, - String description, String dataType, String module) { + public Parameter( + String name, + String defaultValue, + String comment, + String description, + String dataType, + String module) { this(name, defaultValue, comment, description, dataType, module, null, null); } - public Parameter(String name, String defaultValue, String comment, String description, - String dataType, String module, List candidateValues) { + public Parameter( + String name, + String defaultValue, + String comment, + String description, + String dataType, + String module, + List candidateValues) { this(name, defaultValue, comment, description, dataType, module, candidateValues, null); } - public Parameter(String name, String defaultValue, String comment, String description, - String dataType, String module, List candidateValues, List dependencies) { + public Parameter( + String name, + String defaultValue, + String comment, + String description, + String dataType, + String module, + List candidateValues, + List dependencies) { this.name = name; this.defaultValue = defaultValue; this.comment = comment; @@ -105,4 +105,4 @@ public class Parameter { private List includesValue; } } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java b/common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java index 798c84450..4cead0ddb 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.common.pojo; -import java.util.List; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; +import java.util.List; + @AllArgsConstructor @Data @ToString @@ -21,4 +21,4 @@ public class QueryAuthorization { public QueryAuthorization(String message) { this.message = message; } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/RecordInfo.java b/common/src/main/java/com/tencent/supersonic/common/pojo/RecordInfo.java index 00e6362eb..c3c52e2c0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/RecordInfo.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/RecordInfo.java @@ -41,8 +41,7 @@ public class RecordInfo { return false; } RecordInfo that = (RecordInfo) o; - return Objects.equal(createdBy, that.createdBy) && Objects.equal( - updatedBy, that.updatedBy); + return Objects.equal(createdBy, that.createdBy) && Objects.equal(updatedBy, that.updatedBy); } @Override diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java index d687564c9..a799db1bd 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java @@ -5,9 +5,7 @@ import com.tencent.supersonic.common.util.TraceIdUtil; import lombok.Data; import org.slf4j.MDC; -/*** - * result data - */ +/** * result data */ @Data public class ResultData { private int code; @@ -36,5 +34,4 @@ public class ResultData { resultData.setTraceId(MDC.get(TraceIdUtil.TRACE_ID)); return resultData; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java index 5fe7b27da..e24ab979a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java @@ -20,5 +20,4 @@ public class Text2SQLExemplar { private String dbSchema; private String sql; - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java index 3227828c7..2ce4c72c8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java @@ -17,14 +17,11 @@ public class ThreadPoolExecutorMdcWrapper extends ThreadPoolTaskExecutor { @Override public Future submit(Callable task) { - return super - .submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + return super.submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); } @Override public Future submit(Runnable task) { - return super - .submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + return super.submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); } } - diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java index e438380ae..91e23b6a8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum AggOperatorEnum { - MAX("MAX"), MIN("MIN"), @@ -43,14 +42,14 @@ public enum AggOperatorEnum { } /** - * Determine if aggType is count_Distinct type - * 1.outer SQL parses the count_distinct(field) operator as count(DISTINCT field). - * 2.tableSQL generates aggregation that ignores the count_distinct operator. + * Determine if aggType is count_Distinct type 1.outer SQL parses the count_distinct(field) + * operator as count(DISTINCT field). 2.tableSQL generates aggregation that ignores the + * count_distinct operator. + * * @param aggType aggType * @return is count_Distinct type or not */ public static boolean isCountDistinct(String aggType) { return null != aggType && aggType.toUpperCase().equals(COUNT_DISTINCT.getOperator()); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ApiItemType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ApiItemType.java index 1b47cfbe8..c448fd518 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ApiItemType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ApiItemType.java @@ -1,9 +1,7 @@ package com.tencent.supersonic.common.pojo.enums; public enum ApiItemType { - METRIC, TAG, DIMENSION - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java index 80d7ba7af..a049a4c77 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java @@ -1,8 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum AuthType { - VISIBLE, ADMIN - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ConfigMode.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ConfigMode.java index 2549469e3..ca2a4f86a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ConfigMode.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ConfigMode.java @@ -1,8 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum ConfigMode { - - DETAIL("DETAIL"), AGG("AGG"), UNKNOWN("UNKNOWN"); diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataFormatTypeEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataFormatTypeEnum.java index a4c5b5564..ace34beb6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataFormatTypeEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataFormatTypeEnum.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum DataFormatTypeEnum { - PERCENT("percent"), DECIMAL("decimal"); @@ -15,5 +14,4 @@ public enum DataFormatTypeEnum { public String getName() { return name; } - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataTypeEnums.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataTypeEnums.java index 8e4eb25c8..c17fda72a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataTypeEnums.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataTypeEnums.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum DataTypeEnums { - ARRAY("ARRAY"), MAP("MAP"), @@ -42,4 +41,4 @@ public enum DataTypeEnums { } return DataTypeEnums.UNKNOWN; } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DatePeriodEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DatePeriodEnum.java index e7dc34989..de6a25fef 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DatePeriodEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DatePeriodEnum.java @@ -24,5 +24,4 @@ public enum DatePeriodEnum { } return null; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java index 7d1074d4e..28b5b50ea 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java @@ -1,14 +1,11 @@ package com.tencent.supersonic.common.pojo.enums; import org.apache.commons.lang3.StringUtils; + import java.util.Objects; -/*** - * nature type - * such as : metric、dimension etc. - */ +/** * nature type such as : metric、dimension etc. */ public enum DictWordType { - METRIC("metric"), DIMENSION("dimension"), @@ -48,13 +45,15 @@ public enum DictWordType { return dictWordType; } } - //dataSet + // dataSet String[] natures = nature.split(DictWordType.NATURE_SPILT); if (natures.length == 2 && StringUtils.isNumeric(natures[1])) { return DATASET; } - //dimension value - if (natures.length == 3 && StringUtils.isNumeric(natures[1]) && StringUtils.isNumeric(natures[2])) { + // dimension value + if (natures.length == 3 + && StringUtils.isNumeric(natures[1]) + && StringUtils.isNumeric(natures[2])) { return VALUE; } return null; @@ -76,5 +75,4 @@ public enum DictWordType { } return ""; } - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ErrorCode.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ErrorCode.java index 1a19b31be..3762e0243 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ErrorCode.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/ErrorCode.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum ErrorCode { - MULTIPLE_ERRORS_PLACEHOLDER, MULTIPLE_ERRORS, NULL_POINTER, @@ -12,6 +11,5 @@ public enum ErrorCode { DUPLICATED_THEME, UNKNOWN; - private ErrorCode() { - } + private ErrorCode() {} } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java index f91ceca6c..fb4968c68 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java @@ -1,9 +1,7 @@ package com.tencent.supersonic.common.pojo.enums; public enum EventType { - ADD, UPDATE, DELETE - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java index 86b499667..d54e4996d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.pojo.enums; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; @@ -35,7 +34,8 @@ public enum FilterOperatorEnum { @JsonCreator public static FilterOperatorEnum getSqlOperator(String type) { for (FilterOperatorEnum operatorEnum : FilterOperatorEnum.values()) { - if (operatorEnum.value.equalsIgnoreCase(type) || operatorEnum.name().equalsIgnoreCase(type)) { + if (operatorEnum.value.equalsIgnoreCase(type) + || operatorEnum.name().equalsIgnoreCase(type)) { return operatorEnum; } } @@ -48,9 +48,12 @@ public enum FilterOperatorEnum { } public static boolean isValueCompare(FilterOperatorEnum filterOperatorEnum) { - return EQUALS.equals(filterOperatorEnum) || GREATER_THAN.equals(filterOperatorEnum) - || GREATER_THAN_EQUALS.equals(filterOperatorEnum) || MINOR_THAN.equals(filterOperatorEnum) - || MINOR_THAN_EQUALS.equals(filterOperatorEnum) || NOT_EQUALS.equals(filterOperatorEnum); + return EQUALS.equals(filterOperatorEnum) + || GREATER_THAN.equals(filterOperatorEnum) + || GREATER_THAN_EQUALS.equals(filterOperatorEnum) + || MINOR_THAN.equals(filterOperatorEnum) + || MINOR_THAN_EQUALS.equals(filterOperatorEnum) + || NOT_EQUALS.equals(filterOperatorEnum); } public static ComparisonOperator createExpression(FilterOperatorEnum operator) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java index a92524b29..49c708e0c 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.common.pojo.enums; public enum FilterType { - //filtering between different dimensions will directly splice the AND clause + // filtering between different dimensions will directly splice the AND clause AND, - //filtering between different dimensions will generate multiple sql clauses and splice them together using union + // filtering between different dimensions will generate multiple sql clauses and splice them + // together using union UNION } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/PublishEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/PublishEnum.java index dd735ce9b..f5777c30f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/PublishEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/PublishEnum.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum PublishEnum { - UN_PUBLISHED(0), PUBLISHED(1); @@ -14,5 +13,4 @@ public enum PublishEnum { public Integer getCode() { return code; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java index ea1a00939..b55ae6e54 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java @@ -1,20 +1,12 @@ package com.tencent.supersonic.common.pojo.enums; -/** - * Enumerate query types supported by SuperSonic. - */ +/** Enumerate query types supported by SuperSonic. */ public enum QueryType { - /** - * queries with metric calculation (optionally slice and dice by dimensions) - */ + /** queries with metric calculation (optionally slice and dice by dimensions) */ METRIC, - /** - * queries with tag-based entity targeting - */ + /** queries with tag-based entity targeting */ DETAIL, - /** - * queries with ID-based entity selection - */ + /** queries with ID-based entity selection */ ID; public boolean isNativeAggQuery() { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SensitiveLevelEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SensitiveLevelEnum.java index 9e19792de..1ecb5bd05 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SensitiveLevelEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SensitiveLevelEnum.java @@ -1,12 +1,10 @@ package com.tencent.supersonic.common.pojo.enums; public enum SensitiveLevelEnum { - LOW(0), MID(1), HIGH(2); - private Integer code; SensitiveLevelEnum(Integer code) { @@ -16,5 +14,4 @@ public enum SensitiveLevelEnum { public Integer getCode() { return code; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SinkDbEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SinkDbEnum.java index 072aec700..b5e94996a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SinkDbEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/SinkDbEnum.java @@ -1,17 +1,14 @@ package com.tencent.supersonic.common.pojo.enums; public enum SinkDbEnum { - TDW("TDW"), DORIS("DORIS"), ICEBERY("ICEBERY"), - NOT_SUPPORT("NOT_SUPPORT"); - private String db; SinkDbEnum(String db) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/StatusEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/StatusEnum.java index b7175cd0f..168e1eda9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/StatusEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/StatusEnum.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum StatusEnum { - INITIALIZED("INITIALIZED", 0), ONLINE("ONLINE", 1), OFFLINE("OFFLINE", 2), @@ -9,7 +8,6 @@ public enum StatusEnum { UNAVAILABLE("UNAVAILABLE", 4), UNKNOWN("UNKNOWN", -1); - private String status; private Integer code; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java index 6bea6a6cd..5ebfd6816 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum TaskStatusEnum { - INITIAL("initial", -2), ERROR("error", -1), @@ -47,5 +46,4 @@ public enum TaskStatusEnum { } return TaskStatusEnum.UNKNOWN; } - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/Text2SQLType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/Text2SQLType.java index b40f06b73..89c107577 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/Text2SQLType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/Text2SQLType.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.common.pojo.enums; public enum Text2SQLType { - - ONLY_RULE, ONLY_LLM, RULE_AND_LLM; + ONLY_RULE, + ONLY_LLM, + RULE_AND_LLM; public boolean enableRule() { return this.equals(ONLY_RULE) || this.equals(RULE_AND_LLM); @@ -11,5 +12,4 @@ public enum Text2SQLType { public boolean enableLLM() { return this.equals(ONLY_LLM) || this.equals(RULE_AND_LLM); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java index 9ef0a53a8..22502c7bd 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java @@ -7,9 +7,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - public enum TimeDimensionEnum { - DAY("sys_imp_date", "数据日期"), WEEK("sys_imp_week", "数据日期_周"), @@ -33,21 +31,33 @@ public enum TimeDimensionEnum { } public static List getNameList() { - return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName).collect(Collectors.toList()); + return Arrays.stream(TimeDimensionEnum.values()) + .map(TimeDimensionEnum::getName) + .collect(Collectors.toList()); } public static List getChNameList() { - return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getChName).collect(Collectors.toList()); + return Arrays.stream(TimeDimensionEnum.values()) + .map(TimeDimensionEnum::getChName) + .collect(Collectors.toList()); } public static Map getChNameToNameMap() { return Arrays.stream(TimeDimensionEnum.values()) - .collect(Collectors.toMap(TimeDimensionEnum::getChName, TimeDimensionEnum::getName, (k1, k2) -> k1)); + .collect( + Collectors.toMap( + TimeDimensionEnum::getChName, + TimeDimensionEnum::getName, + (k1, k2) -> k1)); } public static Map getNameToNameMap() { return Arrays.stream(TimeDimensionEnum.values()) - .collect(Collectors.toMap(TimeDimensionEnum::getName, TimeDimensionEnum::getName, (k1, k2) -> k1)); + .collect( + Collectors.toMap( + TimeDimensionEnum::getName, + TimeDimensionEnum::getName, + (k1, k2) -> k1)); } public String getName() { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java index 578768655..6fa0bf9a4 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java @@ -2,10 +2,7 @@ package com.tencent.supersonic.common.pojo.enums; public enum TimeMode { - /** - * date mode - * LAST - a certain time - * RECENT - a period time - */ - LAST, RECENT + /** date mode LAST - a certain time RECENT - a period time */ + LAST, + RECENT } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java index 65d356b37..f78b2dee8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo.enums; public enum TypeEnums { - METRIC, DIMENSION, TAG_OBJECT, @@ -11,5 +10,4 @@ public enum TypeEnums { DATASET, MODEL, UNKNOWN - } diff --git a/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java b/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java index b8479d952..12e1c6dd6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java +++ b/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java @@ -13,8 +13,7 @@ import org.springframework.web.bind.annotation.RestController; @RequestMapping({"/api/semantic/parameter"}) public class SystemConfigController { - @Autowired - private SystemConfigService sysConfigService; + @Autowired private SystemConfigService sysConfigService; @PostMapping public Boolean save(@RequestBody SystemConfig systemConfig) { @@ -26,5 +25,4 @@ public class SystemConfigController { public SystemConfig get() { return sysConfigService.getSystemConfig(); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/EmbeddingService.java b/common/src/main/java/com/tencent/supersonic/common/service/EmbeddingService.java index 60d899821..c8cfc9181 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/EmbeddingService.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/EmbeddingService.java @@ -7,8 +7,8 @@ import dev.langchain4j.store.embedding.RetrieveQueryResult; import java.util.List; /** - * Supersonic EmbeddingStore - * Enhanced the functionality by enabling the addition and querying of collection names. + * Supersonic EmbeddingStore Enhanced the functionality by enabling the addition and querying of + * collection names. */ public interface EmbeddingService { @@ -16,6 +16,6 @@ public interface EmbeddingService { void deleteQuery(String collectionName, List queries); - List retrieveQuery(String collectionName, RetrieveQuery retrieveQuery, int num); - + List retrieveQuery( + String collectionName, RetrieveQuery retrieveQuery, int num); } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/ExemplarService.java b/common/src/main/java/com/tencent/supersonic/common/service/ExemplarService.java index 69a21fe16..2706274b7 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/ExemplarService.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/ExemplarService.java @@ -12,5 +12,4 @@ public interface ExemplarService { List recallExemplars(String collection, String query, int num); List recallExemplars(String query, int num); - } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java b/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java index 57506dcd4..46c9a4a85 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java @@ -1,13 +1,12 @@ package com.tencent.supersonic.common.service; import com.baomidou.mybatisplus.extension.service.IService; -import com.tencent.supersonic.common.persistence.dataobject.SystemConfigDO; import com.tencent.supersonic.common.config.SystemConfig; +import com.tencent.supersonic.common.persistence.dataobject.SystemConfigDO; public interface SystemConfigService extends IService { SystemConfig getSystemConfig(); void save(SystemConfig systemConfig); - } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java index b75efd5ed..316980d8a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java @@ -38,10 +38,11 @@ import java.util.stream.Collectors; @Slf4j public class EmbeddingServiceImpl implements EmbeddingService { - private Cache cache = CacheBuilder.newBuilder() - .maximumSize(10000) - .expireAfterWrite(10, TimeUnit.HOURS) - .build(); + private Cache cache = + CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterWrite(10, TimeUnit.HOURS) + .build(); @Override public void addQuery(String collectionName, List queries) { @@ -59,13 +60,17 @@ public class EmbeddingServiceImpl implements EmbeddingService { } embeddingStore.add(embedding, query); } catch (Exception e) { - log.error("embeddingModel embed error question: {}, embeddingStore: {}", question, - embeddingStore.getClass().getSimpleName(), e); + log.error( + "embeddingModel embed error question: {}, embeddingStore: {}", + question, + embeddingStore.getClass().getSimpleName(), + e); } } } - private boolean existSegment(EmbeddingStore embeddingStore, TextSegment query, Embedding embedding) { + private boolean existSegment( + EmbeddingStore embeddingStore, TextSegment query, Embedding embedding) { String queryId = TextSegmentConvert.getQueryId(query); if (queryId == null) { return false; @@ -78,8 +83,13 @@ public class EmbeddingServiceImpl implements EmbeddingService { Map filterCondition = new HashMap<>(); filterCondition.put(TextSegmentConvert.QUERY_ID, queryId); Filter filter = createCombinedFilter(filterCondition); - EmbeddingSearchRequest request = EmbeddingSearchRequest.builder() - .queryEmbedding(embedding).filter(filter).minScore(1.0d).maxResults(1).build(); + EmbeddingSearchRequest request = + EmbeddingSearchRequest.builder() + .queryEmbedding(embedding) + .filter(filter) + .minScore(1.0d) + .maxResults(1) + .build(); EmbeddingSearchResult result = embeddingStore.search(request); List> relevant = result.matches(); @@ -90,16 +100,21 @@ public class EmbeddingServiceImpl implements EmbeddingService { @Override public void deleteQuery(String collectionName, List queries) { - //Not supported yet in Milvus and Chroma + // Not supported yet in Milvus and Chroma EmbeddingStoreFactory embeddingStoreFactory = EmbeddingStoreFactoryProvider.getFactory(); EmbeddingStore embeddingStore = embeddingStoreFactory.create(collectionName); try { if (embeddingStore instanceof InMemoryEmbeddingStore) { - InMemoryEmbeddingStore inMemoryEmbeddingStore = (InMemoryEmbeddingStore) embeddingStore; - List queryIds = queries.stream().map(textSegment -> TextSegmentConvert.getQueryId(textSegment)) - .filter(Objects::nonNull).collect(Collectors.toList()); + InMemoryEmbeddingStore inMemoryEmbeddingStore = + (InMemoryEmbeddingStore) embeddingStore; + List queryIds = + queries.stream() + .map(textSegment -> TextSegmentConvert.getQueryId(textSegment)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(queryIds)) { - MetadataFilterBuilder filterBuilder = new MetadataFilterBuilder(TextSegmentConvert.QUERY_ID); + MetadataFilterBuilder filterBuilder = + new MetadataFilterBuilder(TextSegmentConvert.QUERY_ID); Filter filter = filterBuilder.isIn(queryIds); inMemoryEmbeddingStore.removeAll(filter); } @@ -112,31 +127,46 @@ public class EmbeddingServiceImpl implements EmbeddingService { } @Override - public List retrieveQuery(String collectionName, RetrieveQuery retrieveQuery, int num) { - EmbeddingStore embeddingStore = EmbeddingStoreFactoryProvider.getFactory().create(collectionName); + public List retrieveQuery( + String collectionName, RetrieveQuery retrieveQuery, int num) { + EmbeddingStore embeddingStore = + EmbeddingStoreFactoryProvider.getFactory().create(collectionName); EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(); Map filterCondition = retrieveQuery.getFilterCondition(); return retrieveQuery.getQueryTextsList().stream() - .map(queryText -> retrieveSingleQuery(queryText, embeddingModel, embeddingStore, filterCondition, num)) + .map( + queryText -> + retrieveSingleQuery( + queryText, + embeddingModel, + embeddingStore, + filterCondition, + num)) .collect(Collectors.toList()); } - private RetrieveQueryResult retrieveSingleQuery(String queryText, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, - Map filterCondition, - int num) { + private RetrieveQueryResult retrieveSingleQuery( + String queryText, + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore, + Map filterCondition, + int num) { Embedding embeddedText = embeddingModel.embed(queryText).content(); Filter filter = createCombinedFilter(filterCondition); - EmbeddingSearchRequest request = EmbeddingSearchRequest.builder() - .queryEmbedding(embeddedText).filter(filter).maxResults(num).build(); + EmbeddingSearchRequest request = + EmbeddingSearchRequest.builder() + .queryEmbedding(embeddedText) + .filter(filter) + .maxResults(num) + .build(); EmbeddingSearchResult result = embeddingStore.search(request); - List retrievals = result.matches().stream() - .map(this::convertToRetrieval) - .sorted(Comparator.comparingDouble(Retrieval::getDistance).reversed()) - .limit(num) - .collect(Collectors.toList()); + List retrievals = + result.matches().stream() + .map(this::convertToRetrieval) + .sorted(Comparator.comparingDouble(Retrieval::getDistance).reversed()) + .limit(num) + .collect(Collectors.toList()); RetrieveQueryResult retrieveQueryResult = new RetrieveQueryResult(); retrieveQueryResult.setQuery(queryText); diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java index 8483a8ff8..7b4eb92a7 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java @@ -31,20 +31,19 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner { private static final String SYS_EXEMPLAR_FILE = "s2-exemplar.json"; - private TypeReference> valueTypeRef = new TypeReference>() { - }; + private TypeReference> valueTypeRef = + new TypeReference>() {}; private final ObjectMapper objectMapper = JsonUtil.INSTANCE.getObjectMapper(); - @Autowired - private EmbeddingConfig embeddingConfig; + @Autowired private EmbeddingConfig embeddingConfig; - @Autowired - private EmbeddingService embeddingService; + @Autowired private EmbeddingService embeddingService; public void storeExemplar(String collection, Text2SQLExemplar exemplar) { - Metadata metadata = Metadata.from(JsonUtil.toMap(JsonUtil.toString(exemplar), - String.class, Object.class)); + Metadata metadata = + Metadata.from( + JsonUtil.toMap(JsonUtil.toString(exemplar), String.class, Object.class)); TextSegment segment = TextSegment.from(exemplar.getQuestion(), metadata); TextSegmentConvert.addQueryId(segment, exemplar.getQuestion()); @@ -52,8 +51,9 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner { } public void removeExemplar(String collection, Text2SQLExemplar exemplar) { - Metadata metadata = Metadata.from(JsonUtil.toMap(JsonUtil.toString(exemplar), - String.class, Object.class)); + Metadata metadata = + Metadata.from( + JsonUtil.toMap(JsonUtil.toString(exemplar), String.class, Object.class)); TextSegment segment = TextSegment.from(exemplar.getQuestion(), metadata); embeddingService.deleteQuery(collection, Lists.newArrayList(segment)); @@ -66,15 +66,22 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner { public List recallExemplars(String collection, String query, int num) { List exemplars = Lists.newArrayList(); - RetrieveQuery retrieveQuery = RetrieveQuery.builder() - .queryTextsList(Lists.newArrayList(query)) - .build(); - List results = embeddingService.retrieveQuery(collection, retrieveQuery, num); - results.stream().forEach(ret -> { - ret.getRetrieval().stream().forEach(r -> { - exemplars.add(JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class)); - }); - }); + RetrieveQuery retrieveQuery = + RetrieveQuery.builder().queryTextsList(Lists.newArrayList(query)).build(); + List results = + embeddingService.retrieveQuery(collection, retrieveQuery, num); + results.stream() + .forEach( + ret -> { + ret.getRetrieval().stream() + .forEach( + r -> { + exemplars.add( + JsonUtil.mapToObject( + r.getMetadata(), + Text2SQLExemplar.class)); + }); + }); return exemplars; } @@ -95,5 +102,4 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner { String collection = embeddingConfig.getText2sqlCollectionName(); exemplars.stream().forEach(e -> storeExemplar(collection, e)); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java index 9509cc734..4b48331ad 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java @@ -18,11 +18,10 @@ import java.util.List; import java.util.concurrent.atomic.AtomicReference; @Service -public class SystemConfigServiceImpl - extends ServiceImpl implements SystemConfigService { +public class SystemConfigServiceImpl extends ServiceImpl + implements SystemConfigService { - @Autowired - private Environment environment; + @Autowired private Environment environment; // Cache field to store the system configuration private AtomicReference cachedSystemConfig = new AtomicReference<>(); @@ -45,11 +44,13 @@ public class SystemConfigServiceImpl systemConfig.setId(1); systemConfig.init(); // use system property to initialize system parameter - systemConfig.getParameters().stream().forEach(p -> { - if (environment.containsProperty(p.getName())) { - p.setValue(environment.getProperty(p.getName())); - } - }); + systemConfig.getParameters().stream() + .forEach( + p -> { + if (environment.containsProperty(p.getName())) { + p.setValue(environment.getProperty(p.getName())); + } + }); save(systemConfig); return systemConfig; } @@ -67,9 +68,9 @@ public class SystemConfigServiceImpl private SystemConfig convert(SystemConfigDO systemConfigDO) { SystemConfig sysParameter = new SystemConfig(); sysParameter.setId(systemConfigDO.getId()); - List parameters = JsonUtil.toObject(systemConfigDO.getParameters(), - new TypeReference>() { - }); + List parameters = + JsonUtil.toObject( + systemConfigDO.getParameters(), new TypeReference>() {}); sysParameter.setParameters(parameters); sysParameter.setAdminList(systemConfigDO.getAdmin()); return sysParameter; @@ -82,5 +83,4 @@ public class SystemConfigServiceImpl sysParameterDO.setAdmin(sysParameter.getAdmin()); return sysParameterDO; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/AESEncryptionUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/AESEncryptionUtil.java index 7a8eda1e0..ffae97722 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/AESEncryptionUtil.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/AESEncryptionUtil.java @@ -1,12 +1,13 @@ package com.tencent.supersonic.common.util; -import lombok.extern.slf4j.Slf4j; - import javax.crypto.Cipher; import javax.crypto.SecretKeyFactory; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; + +import lombok.extern.slf4j.Slf4j; + import java.security.MessageDigest; import java.security.spec.KeySpec; import java.util.Arrays; @@ -20,7 +21,8 @@ public class AESEncryptionUtil { private static final String SECRET_KEY_ALGORITHM = "PBKDF2WithHmacSHA256"; private static final int ITERATIONS = 65536; private static final int KEY_LENGTH = 256; - private static final String KEY = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"; + private static final String KEY = + "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"; // TODO 固定IV，确保每次加密时使用相同的IV,该值应该安全保管 private static final String IV = "supersonic@bicom"; @@ -120,8 +122,10 @@ public class AESEncryptionUtil { int len = hexString.length(); byte[] byteArray = new byte[len / 2]; for (int i = 0; i < len; i += 2) { - byteArray[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4) - + Character.digit(hexString.charAt(i + 1), 16)); + byteArray[i / 2] = + (byte) + ((Character.digit(hexString.charAt(i), 16) << 4) + + Character.digit(hexString.charAt(i + 1), 16)); } return byteArray; } @@ -133,5 +137,4 @@ public class AESEncryptionUtil { public static byte[] getBytesFromString(String str) { return Base64.getDecoder().decode(str); } - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/BeanMapper.java b/common/src/main/java/com/tencent/supersonic/common/util/BeanMapper.java index 9f7206f2e..684e5ddfa 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/BeanMapper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/BeanMapper.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.common.util; -import java.beans.PropertyDescriptor; -import java.util.HashSet; -import java.util.Set; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanWrapper; import org.springframework.beans.BeanWrapperImpl; +import java.beans.PropertyDescriptor; +import java.util.HashSet; +import java.util.Set; public class BeanMapper { @@ -19,7 +19,7 @@ public class BeanMapper { PropertyDescriptor[] pds = src.getPropertyDescriptors(); Set emptyNames = new HashSet(); for (java.beans.PropertyDescriptor pd : pds) { - //check if value of this property is null then add it to the collection + // check if value of this property is null then add it to the collection Object srcValue = src.getPropertyValue(pd.getName()); if (srcValue == null) { emptyNames.add(pd.getName()); @@ -28,5 +28,4 @@ public class BeanMapper { String[] result = new String[emptyNames.size()]; return (String[]) emptyNames.toArray(result); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/ContextUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/ContextUtils.java index 5d48525b4..541fb5e1f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/ContextUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/ContextUtils.java @@ -1,12 +1,13 @@ package com.tencent.supersonic.common.util; -import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; +import java.util.Map; + @Component public class ContextUtils implements ApplicationContextAware { @@ -88,5 +89,4 @@ public class ContextUtils implements ApplicationContextAware { } return activeProfile; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java index aaab270d4..3f63ff2d0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java @@ -30,7 +30,6 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; import static com.tencent.supersonic.common.pojo.Constants.WEEK; import static com.tencent.supersonic.common.pojo.Constants.YEAR; - @Slf4j @Component @Data @@ -38,19 +37,24 @@ public class DateModeUtils { @Value("${s2.query.parameter.sys.date:sys_imp_date}") private String sysDateCol; + @Value("${s2.query.parameter.sys.month:sys_imp_month}") private String sysDateMonthCol; + @Value("${s2.query.parameter.sys.month:sys_imp_week}") private String sysDateWeekCol; @Value("${s2.query.parameter.sys.zipper.begin:start_}") private String sysZipperDateColBegin; + @Value("${s2.query.parameter.sys.zipper.end:end_}") private String sysZipperDateColEnd; public Boolean recentMode(DateConf dateInfo) { - if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT == dateInfo.getDateMode() - && DAY.equalsIgnoreCase(dateInfo.getPeriod()) && Objects.nonNull(dateInfo.getUnit())) { + if (Objects.nonNull(dateInfo) + && DateConf.DateMode.RECENT == dateInfo.getDateMode() + && DAY.equalsIgnoreCase(dateInfo.getPeriod()) + && Objects.nonNull(dateInfo.getUnit())) { return true; } return false; @@ -73,10 +77,10 @@ public class DateModeUtils { public String hasDataModeStr(ItemDateResp dateDate, DateConf dateInfo) { if (Objects.isNull(dateDate) || StringUtils.isEmpty(dateDate.getStartDate()) - || StringUtils.isEmpty(dateDate.getStartDate()) - ) { - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(), sysDateCol, - dateInfo.getEndDate()); + || StringUtils.isEmpty(dateDate.getStartDate())) { + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateCol, dateInfo.getStartDate(), sysDateCol, dateInfo.getEndDate()); } else { log.info("dateDate:{}", dateDate); } @@ -90,21 +94,31 @@ public class DateModeUtils { if (endReq.isAfter(endData)) { if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) { - Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateFormatStr, ChronoUnit.DAYS); + Long unit = + getInterval( + dateInfo.getStartDate(), + dateInfo.getEndDate(), + dateFormatStr, + ChronoUnit.DAYS); LocalDate dateMax = endData; LocalDate dateMin = dateMax.minusDays(unit - 1); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, dateMax); + return String.format( + "(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, dateMax); } if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) { - Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateFormatStr, - ChronoUnit.MONTHS); + Long unit = + getInterval( + dateInfo.getStartDate(), + dateInfo.getEndDate(), + dateFormatStr, + ChronoUnit.MONTHS); return generateMonthSql(endData, unit, dateFormatStr); } - } - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(), sysDateCol, - dateInfo.getEndDate()); + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateCol, dateInfo.getStartDate(), sysDateCol, dateInfo.getEndDate()); } public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr) { @@ -130,7 +144,9 @@ public class DateModeUtils { public String recentDayStr(ItemDateResp dateDate, DateConf dateInfo) { ImmutablePair dayRange = recentDay(dateDate, dateInfo); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dayRange.left, sysDateCol, dayRange.right); + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateCol, dayRange.left, sysDateCol, dayRange.right); } public ImmutablePair recentDay(ItemDateResp dateDate, DateConf dateInfo) { @@ -151,14 +167,16 @@ public class DateModeUtils { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr); String endStr = endData.format(formatter); String start = endData.minusMonths(unit).format(formatter); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol, endStr); + return String.format( + "(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol, endStr); } public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) { List> range = recentMonth(dateDate, dateInfo); if (range.size() == 1) { - return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, range.get(0).left, sysDateMonthCol, - range.get(0).right); + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateMonthCol, range.get(0).left, sysDateMonthCol, range.get(0).right); } if (range.size() > 0) { StringJoiner joiner = new StringJoiner(","); @@ -168,13 +186,20 @@ public class DateModeUtils { return ""; } - public List> recentMonth(ItemDateResp dateDate, DateConf dateInfo) { - LocalDate endData = LocalDate.parse(dateDate.getEndDate(), - DateTimeFormatter.ofPattern(dateDate.getDateFormat())); + public List> recentMonth( + ItemDateResp dateDate, DateConf dateInfo) { + LocalDate endData = + LocalDate.parse( + dateDate.getEndDate(), + DateTimeFormatter.ofPattern(dateDate.getDateFormat())); List> ret = new ArrayList<>(); if (dateDate.getDatePeriod() != null && MONTH.equalsIgnoreCase(dateDate.getDatePeriod())) { - Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateDate.getDateFormat(), - ChronoUnit.MONTHS); + Long unit = + getInterval( + dateInfo.getStartDate(), + dateInfo.getEndDate(), + dateDate.getDateFormat(), + ChronoUnit.MONTHS); LocalDate dateMax = endData; List months = generateMonthStr(dateMax, unit, dateDate.getDateFormat()); if (!CollectionUtils.isEmpty(months)) { @@ -194,14 +219,16 @@ public class DateModeUtils { public String recentWeekStr(LocalDate endData, Long unit) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT); String start = endData.minusDays(unit * 7).format(formatter); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol, - endData.format(formatter)); + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateWeekCol, start, sysDateWeekCol, endData.format(formatter)); } public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) { ImmutablePair dayRange = recentWeek(dateDate, dateInfo); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, dayRange.left, sysDateWeekCol, - dayRange.right); + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateWeekCol, dayRange.left, sysDateWeekCol, dayRange.right); } public ImmutablePair recentWeek(ItemDateResp dateDate, DateConf dateInfo) { @@ -216,7 +243,8 @@ public class DateModeUtils { return ImmutablePair.of(start, end.format(formatter)); } - private Long getInterval(String startDate, String endDate, String dateFormat, ChronoUnit chronoUnit) { + private Long getInterval( + String startDate, String endDate, String dateFormat, ChronoUnit chronoUnit) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormat); try { LocalDate start = LocalDate.parse(startDate, formatter); @@ -254,22 +282,33 @@ public class DateModeUtils { if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) { // startDate YYYYMM if (!dateInfo.getStartDate().contains(Constants.MINUS)) { - return String.format("%s >= '%s' and %s <= '%s'", - sysDateMonthCol, dateInfo.getStartDate(), sysDateMonthCol, dateInfo.getEndDate()); + return String.format( + "%s >= '%s' and %s <= '%s'", + sysDateMonthCol, + dateInfo.getStartDate(), + sysDateMonthCol, + dateInfo.getEndDate()); } - LocalDate endData = LocalDate.parse(dateInfo.getEndDate(), - DateTimeFormatter.ofPattern(DAY_FORMAT)); - LocalDate startData = LocalDate.parse(dateInfo.getStartDate(), - DateTimeFormatter.ofPattern(DAY_FORMAT)); + LocalDate endData = + LocalDate.parse(dateInfo.getEndDate(), DateTimeFormatter.ofPattern(DAY_FORMAT)); + LocalDate startData = + LocalDate.parse( + dateInfo.getStartDate(), DateTimeFormatter.ofPattern(DAY_FORMAT)); DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT); - return String.format("%s >= '%s' and %s <= '%s'", - sysDateMonthCol, startData.format(formatter), sysDateMonthCol, endData.format(formatter)); + return String.format( + "%s >= '%s' and %s <= '%s'", + sysDateMonthCol, + startData.format(formatter), + sysDateMonthCol, + endData.format(formatter)); } if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) { - return String.format("%s >= '%s' and %s <= '%s'", + return String.format( + "%s >= '%s' and %s <= '%s'", sysDateWeekCol, dateInfo.getStartDate(), sysDateWeekCol, dateInfo.getEndDate()); } - return String.format("%s >= '%s' and %s <= '%s'", + return String.format( + "%s >= '%s' and %s <= '%s'", sysDateCol, dateInfo.getStartDate(), sysDateCol, dateInfo.getEndDate()); } @@ -308,7 +347,8 @@ public class DateModeUtils { if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) { LocalDate dateMax = LocalDate.now().minusDays(1); LocalDate dateMin = dateMax.minusDays(unit - 1); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, dateMax); + return String.format( + "(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, dateMax); } if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) { @@ -324,8 +364,9 @@ public class DateModeUtils { return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT); } - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, LocalDate.now().minusDays(2), sysDateCol, - LocalDate.now().minusDays(1)); + return String.format( + "(%s >= '%s' and %s <= '%s')", + sysDateCol, LocalDate.now().minusDays(2), sysDateCol, LocalDate.now().minusDays(1)); } public String getDateWhereStr(DateConf dateInfo) { @@ -352,7 +393,6 @@ public class DateModeUtils { break; default: break; - } return dateStr; } @@ -386,5 +426,4 @@ public class DateModeUtils { } return ""; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java index c445eef37..25ad7fbb8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java @@ -4,6 +4,7 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import lombok.extern.slf4j.Slf4j; + import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -80,7 +81,8 @@ public class DateUtils { return getBeforeDate(currentDate, intervalDay, datePeriodEnum); } - public static String getBeforeDate(String date, int intervalDay, DatePeriodEnum datePeriodEnum) { + public static String getBeforeDate( + String date, int intervalDay, DatePeriodEnum datePeriodEnum) { DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT); LocalDate currentDate = LocalDate.parse(date, dateTimeFormatter); LocalDate result = null; @@ -91,7 +93,9 @@ public class DateUtils { case WEEK: result = currentDate.minusWeeks(intervalDay); if (intervalDay == 0) { - result = result.with(TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY)); + result = + result.with( + TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY)); } break; case MONTH: @@ -103,13 +107,14 @@ public class DateUtils { case QUARTER: result = currentDate.minusMonths(intervalDay * 3L); if (intervalDay == 0) { - TemporalAdjuster firstDayOfQuarter = temporal -> { - LocalDate tempDate = LocalDate.from(temporal); - int month = tempDate.get(ChronoField.MONTH_OF_YEAR); - int firstMonthOfQuarter = ((month - 1) / 3) * 3 + 1; - return tempDate.with(ChronoField.MONTH_OF_YEAR, firstMonthOfQuarter) - .with(TemporalAdjusters.firstDayOfMonth()); - }; + TemporalAdjuster firstDayOfQuarter = + temporal -> { + LocalDate tempDate = LocalDate.from(temporal); + int month = tempDate.get(ChronoField.MONTH_OF_YEAR); + int firstMonthOfQuarter = ((month - 1) / 3) * 3 + 1; + return tempDate.with(ChronoField.MONTH_OF_YEAR, firstMonthOfQuarter) + .with(TemporalAdjusters.firstDayOfMonth()); + }; result = result.with(firstDayOfQuarter); } break; diff --git a/common/src/main/java/com/tencent/supersonic/common/util/FileUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/FileUtils.java index bae0c2129..dc0ceea26 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/FileUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/FileUtils.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.util; - import java.io.File; import java.util.ArrayList; import java.util.Arrays; @@ -10,9 +9,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -/** - * * tools functions to file - */ +/** * tools functions to file */ public class FileUtils { public static Boolean exit(String path) { @@ -25,8 +22,12 @@ public class FileUtils { return -1; } File file = new File(path); - Optional lastModified = Arrays.stream(file.listFiles()).filter(f -> f.isFile()) - .map(f -> f.lastModified()).sorted(Collections.reverseOrder()).findFirst(); + Optional lastModified = + Arrays.stream(file.listFiles()) + .filter(f -> f.isFile()) + .map(f -> f.lastModified()) + .sorted(Collections.reverseOrder()) + .findFirst(); if (lastModified.isPresent()) { return lastModified.get(); @@ -41,7 +42,8 @@ public class FileUtils { return null; } - public static void scanDirectory(File file, int maxLevel, Map> directories) { + public static void scanDirectory( + File file, int maxLevel, Map> directories) { if (maxLevel < 0) { return; } @@ -78,12 +80,12 @@ public class FileUtils { continue; } String parent = f.getParentFile().getParentFile().getName(); - if (result.containsKey(parent) && result.get(parent).containsKey(f.getParentFile().getName())) { + if (result.containsKey(parent) + && result.get(parent).containsKey(f.getParentFile().getName())) { result.get(parent).get(f.getParentFile().getName()).add(f.getName()); } } } return result; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/HttpClientResult.java b/common/src/main/java/com/tencent/supersonic/common/util/HttpClientResult.java index 694a81995..cbf4ee12c 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/HttpClientResult.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/HttpClientResult.java @@ -2,23 +2,17 @@ package com.tencent.supersonic.common.util; import java.io.Serializable; - public class HttpClientResult implements Serializable { private static final long serialVersionUID = 2168152194164783950L; - /** - * 响应状态码 - */ + /** 响应状态码 */ private int code; - /** - * 响应数据 - */ + /** 响应数据 */ private String content; - public HttpClientResult() { - } + public HttpClientResult() {} public HttpClientResult(int code) { this.code = code; @@ -53,5 +47,4 @@ public class HttpClientResult implements Serializable { public String toString() { return "HttpClientResult [code=" + code + ", content=" + content + "]"; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/HttpClientUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/HttpClientUtils.java index 786dfb20d..fe4a31f91 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/HttpClientUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/HttpClientUtils.java @@ -1,19 +1,5 @@ package com.tencent.supersonic.common.util; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.net.SocketException; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.http.Header; @@ -54,21 +40,28 @@ import org.apache.http.util.CharsetUtils; import org.apache.http.util.EntityUtils; import org.springframework.util.CollectionUtils; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.SocketException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + @Slf4j public class HttpClientUtils { - /*** - * Encoding format. Unified sending encoding format using UTF-8 - */ + /** * Encoding format. Unified sending encoding format using UTF-8 */ private static final String ENCODING = "UTF-8"; - /** - * Default number of connections per route - */ + /** Default number of connections per route */ private static final int DEFAULT_MAX_CONN_PERHOST = 100; - /** - * Maximum number of connections in the entire Connection pool - */ + /** Maximum number of connections in the entire Connection pool */ private static final int DEFAULT_MAX_TOTAL_CONN = 200; private static final int DEFAULT_CONNECTION_TIMEOUT = 150000; @@ -86,70 +79,86 @@ public class HttpClientUtils { try { SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory( - SSLContexts.custom().loadTrustMaterial((chain, authType) -> true).build(), - new String[]{"SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"}, + SSLContexts.custom() + .loadTrustMaterial((chain, authType) -> true) + .build(), + new String[] {"SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"}, null, NoopHostnameVerifier.INSTANCE); - PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager( - RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.getSocketFactory()) - .register("https", sslConnectionSocketFactory).build() - ); + PoolingHttpClientConnectionManager connManager = + new PoolingHttpClientConnectionManager( + RegistryBuilder.create() + .register( + "http", PlainConnectionSocketFactory.getSocketFactory()) + .register("https", sslConnectionSocketFactory) + .build()); connManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONN); connManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONN_PERHOST); - RequestConfig requestConfig = RequestConfig.custom() - // 请求超时时间 - .setConnectTimeout(DEFAULT_CONNECTION_TIMEOUT) - // 等待数据超时时间 - .setSocketTimeout(DEFAULT_READ_TIMEOUT) - // 连接不够用时等待超时时间 - .setConnectionRequestTimeout(DEFAULT_CONN_REQUEST_TIMEOUT) - .build(); + RequestConfig requestConfig = + RequestConfig.custom() + // 请求超时时间 + .setConnectTimeout(DEFAULT_CONNECTION_TIMEOUT) + // 等待数据超时时间 + .setSocketTimeout(DEFAULT_READ_TIMEOUT) + // 连接不够用时等待超时时间 + .setConnectionRequestTimeout(DEFAULT_CONN_REQUEST_TIMEOUT) + .build(); - HttpRequestRetryHandler httpRequestRetryHandler = (exception, executionCount, context) -> { - // 如果已经重试了3次，就放弃 - if (executionCount > 3) { - log.warn("Maximum tries reached, exception would be thrown to outer block"); - return false; - } - if (exception instanceof NoHttpResponseException) { - // 如果服务器丢掉了连接，那么就重试 - log.warn("Retry, No response from server on {} error: {}", executionCount, - exception.getMessage()); - return true; - } else if (exception instanceof SocketException) { - // 如果服务器断开了连接，那么就重试 - log.warn("Retry, No connection from server on {} error: {}", executionCount, - exception.getMessage()); - return true; - } - return false; - }; - - httpClient = HttpClients.custom() - // 设置连接池 - .setConnectionManager(connManager) - // 设置超时时间 - .setDefaultRequestConfig(requestConfig) - // 设置连接存活时间 - .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy() { - @Override - public long getKeepAliveDuration(final HttpResponse response, final HttpContext context) { - long keepAlive = super.getKeepAliveDuration(response, context); - if (keepAlive == -1) { - keepAlive = 5000; - } - return keepAlive; + HttpRequestRetryHandler httpRequestRetryHandler = + (exception, executionCount, context) -> { + // 如果已经重试了3次，就放弃 + if (executionCount > 3) { + log.warn( + "Maximum tries reached, exception would be thrown to outer block"); + return false; } - }) - .setRetryHandler(httpRequestRetryHandler) - // 设置连接存活时间 - .setConnectionTimeToLive(5000L, TimeUnit.MILLISECONDS) - // 关闭无效和空闲的连接 - .evictIdleConnections(5L, TimeUnit.SECONDS) - .build(); + if (exception instanceof NoHttpResponseException) { + // 如果服务器丢掉了连接，那么就重试 + log.warn( + "Retry, No response from server on {} error: {}", + executionCount, + exception.getMessage()); + return true; + } else if (exception instanceof SocketException) { + // 如果服务器断开了连接，那么就重试 + log.warn( + "Retry, No connection from server on {} error: {}", + executionCount, + exception.getMessage()); + return true; + } + return false; + }; + + httpClient = + HttpClients.custom() + // 设置连接池 + .setConnectionManager(connManager) + // 设置超时时间 + .setDefaultRequestConfig(requestConfig) + // 设置连接存活时间 + .setKeepAliveStrategy( + new DefaultConnectionKeepAliveStrategy() { + @Override + public long getKeepAliveDuration( + final HttpResponse response, + final HttpContext context) { + long keepAlive = + super.getKeepAliveDuration(response, context); + if (keepAlive == -1) { + keepAlive = 5000; + } + return keepAlive; + } + }) + .setRetryHandler(httpRequestRetryHandler) + // 设置连接存活时间 + .setConnectionTimeToLive(5000L, TimeUnit.MILLISECONDS) + // 关闭无效和空闲的连接 + .evictIdleConnections(5L, TimeUnit.SECONDS) + .build(); } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException(e); @@ -184,36 +193,45 @@ public class HttpClientUtils { * * @return */ - public static HttpClientResult doPost(String url, + public static HttpClientResult doPost( + String url, String proxyHost, Integer proxyPort, Map headers, Map params) { - return RetryUtils.exec(() -> { - HttpPost httpPost = null; - CloseableHttpResponse response = null; - try { - httpPost = new HttpPost(url); - setProxy(httpPost, proxyHost, proxyPort); + return RetryUtils.exec( + () -> { + HttpPost httpPost = null; + CloseableHttpResponse response = null; + try { + httpPost = new HttpPost(url); + setProxy(httpPost, proxyHost, proxyPort); - // 封装header参数 - packageHeader(headers, httpPost); - // 封装请求参数 - packageParam(params, httpPost); + // 封装header参数 + packageHeader(headers, httpPost); + // 封装请求参数 + packageParam(params, httpPost); - response = httpClient.execute(httpPost); - // 获取返回结果 - HttpClientResult result = getHttpClientResult(response); - log.info("uri:{}, req:{}, resp:{}", url, - "headers:" + getHeaders(httpPost) + "------params:" + params, result); - return result; - } catch (Exception e) { - log.error("uri:{}, req:{}", url, "headers:" + headers + "------params:" + params, e); - throw new RuntimeException(e.getMessage()); - } finally { - close(httpPost, response); - } - }); + response = httpClient.execute(httpPost); + // 获取返回结果 + HttpClientResult result = getHttpClientResult(response); + log.info( + "uri:{}, req:{}, resp:{}", + url, + "headers:" + getHeaders(httpPost) + "------params:" + params, + result); + return result; + } catch (Exception e) { + log.error( + "uri:{}, req:{}", + url, + "headers:" + headers + "------params:" + params, + e); + throw new RuntimeException(e.getMessage()); + } finally { + close(httpPost, response); + } + }); } /** @@ -224,7 +242,8 @@ public class HttpClientUtils { * @param params * @return */ - public static HttpClientResult doPost(String url, Map header, Map params) { + public static HttpClientResult doPost( + String url, Map header, Map params) { return doPost(url, null, null, header, params); } @@ -260,41 +279,53 @@ public class HttpClientUtils { * @return * @throws Exception */ - public static HttpClientResult doGet(String url, String proxyHost, Integer proxyPort, Map headers, + public static HttpClientResult doGet( + String url, + String proxyHost, + Integer proxyPort, + Map headers, Map params) { - return RetryUtils.exec(() -> { - HttpGet httpGet = null; - CloseableHttpResponse response = null; - try { - // 创建访问的地址 - URIBuilder uriBuilder = new URIBuilder(url); - if (params != null) { - Set> entrySet = params.entrySet(); - for (Map.Entry entry : entrySet) { - uriBuilder.setParameter(entry.getKey(), entry.getValue()); + return RetryUtils.exec( + () -> { + HttpGet httpGet = null; + CloseableHttpResponse response = null; + try { + // 创建访问的地址 + URIBuilder uriBuilder = new URIBuilder(url); + if (params != null) { + Set> entrySet = params.entrySet(); + for (Map.Entry entry : entrySet) { + uriBuilder.setParameter(entry.getKey(), entry.getValue()); + } + } + + httpGet = new HttpGet(uriBuilder.build()); + setProxy(httpGet, proxyHost, proxyPort); + + // 设置请求头 + packageHeader(headers, httpGet); + + response = httpClient.execute(httpGet); + + // 获取返回结果 + HttpClientResult res = getHttpClientResult(response); + log.debug( + "GET uri:{}, req:{}, resp:{}", + url, + "headers:" + getHeaders(httpGet) + "------params:" + params, + res); + return res; + } catch (Exception e) { + log.error( + "GET error! uri:{}, req:{}", + url, + "headers:" + headers + "------params:" + params, + e); + throw new RuntimeException(e.getMessage()); + } finally { + close(httpGet, response); } - } - - httpGet = new HttpGet(uriBuilder.build()); - setProxy(httpGet, proxyHost, proxyPort); - - // 设置请求头 - packageHeader(headers, httpGet); - - response = httpClient.execute(httpGet); - - // 获取返回结果 - HttpClientResult res = getHttpClientResult(response); - log.debug("GET uri:{}, req:{}, resp:{}", url, - "headers:" + getHeaders(httpGet) + "------params:" + params, res); - return res; - } catch (Exception e) { - log.error("GET error! uri:{}, req:{}", url, "headers:" + headers + "------params:" + params, e); - throw new RuntimeException(e.getMessage()); - } finally { - close(httpGet, response); - } - }); + }); } /** @@ -305,13 +336,12 @@ public class HttpClientUtils { * @param params * @return */ - public static HttpClientResult doGet(String url, Map header, Map params) { + public static HttpClientResult doGet( + String url, Map header, Map params) { return doGet(url, null, null, header, params); } - /** - * - */ + /** */ public static void close(HttpRequestBase httpRequest, CloseableHttpResponse response) { if (response != null) { try { @@ -324,7 +354,6 @@ public class HttpClientUtils { } catch (Throwable ex) { log.error("response close error : ", ex); } - } if (httpRequest != null) { try { @@ -370,7 +399,8 @@ public class HttpClientUtils { * @param httpMethod * @throws UnsupportedEncodingException */ - public static void packageParam(Map params, HttpEntityEnclosingRequestBase httpMethod) + public static void packageParam( + Map params, HttpEntityEnclosingRequestBase httpMethod) throws UnsupportedEncodingException { if (params != null) { List nvps = new ArrayList(); @@ -386,19 +416,18 @@ public class HttpClientUtils { public static void setProxy(HttpRequestBase httpMethod, String proxyHost, Integer proxyPort) { if (!StringUtils.isEmpty(proxyHost) && proxyPort != null) { - RequestConfig config = RequestConfig.custom() - .setProxy(new HttpHost(proxyHost, proxyPort)) - .setConnectTimeout(10000) - .setSocketTimeout(10000) - .setConnectionRequestTimeout(3000) - .build(); + RequestConfig config = + RequestConfig.custom() + .setProxy(new HttpHost(proxyHost, proxyPort)) + .setConnectTimeout(10000) + .setSocketTimeout(10000) + .setConnectionRequestTimeout(3000) + .build(); httpMethod.setConfig(config); } } - /** - * is respond successfully --200 - */ + /** is respond successfully --200 */ public static boolean isOK(HttpClientResult res) { return res != null && res.getCode() == HttpStatus.SC_OK; } @@ -408,88 +437,111 @@ public class HttpClientUtils { * * @return */ - public static HttpClientResult doPostJSON(String url, String proxyHost, Integer proxyPort, - Map headers, String req) { - return RetryUtils.exec(() -> { - HttpPost httpPost = null; - CloseableHttpResponse response = null; - try { - httpPost = new HttpPost(url); - setProxy(httpPost, proxyHost, proxyPort); + public static HttpClientResult doPostJSON( + String url, + String proxyHost, + Integer proxyPort, + Map headers, + String req) { + return RetryUtils.exec( + () -> { + HttpPost httpPost = null; + CloseableHttpResponse response = null; + try { + httpPost = new HttpPost(url); + setProxy(httpPost, proxyHost, proxyPort); - // 封装header参数 - packageHeader(headers, httpPost); - httpPost.setHeader("Content-Type", "application/json;charset=UTF-8"); + // 封装header参数 + packageHeader(headers, httpPost); + httpPost.setHeader("Content-Type", "application/json;charset=UTF-8"); - // 封装请求参数 - StringEntity stringEntity = new StringEntity(req, ENCODING); // 解决中文乱码问题 - stringEntity.setContentEncoding("UTF-8"); + // 封装请求参数 + StringEntity stringEntity = new StringEntity(req, ENCODING); // 解决中文乱码问题 + stringEntity.setContentEncoding("UTF-8"); - httpPost.setEntity(stringEntity); + httpPost.setEntity(stringEntity); - response = httpClient.execute(httpPost); - // 获取返回结果 - HttpClientResult res = getHttpClientResult(response); - log.info("doPostJSON uri:{}, req:{}, resp:{}", url, - "headers:" + getHeaders(httpPost) + "------req:" + req, res); - return res; - } catch (Exception e) { - log.error("doPostJSON error! uri:{}, req:{}", url, "headers:" + headers + "------req:" + req, e); - throw new RuntimeException(e.getMessage()); - } finally { - close(httpPost, response); - } - }); + response = httpClient.execute(httpPost); + // 获取返回结果 + HttpClientResult res = getHttpClientResult(response); + log.info( + "doPostJSON uri:{}, req:{}, resp:{}", + url, + "headers:" + getHeaders(httpPost) + "------req:" + req, + res); + return res; + } catch (Exception e) { + log.error( + "doPostJSON error! uri:{}, req:{}", + url, + "headers:" + headers + "------req:" + req, + e); + throw new RuntimeException(e.getMessage()); + } finally { + close(httpPost, response); + } + }); } public static HttpClientResult doPostJSON(String url, String req) { return doPostJSON(url, null, null, null, req); } - /** - * get json - */ - public static HttpClientResult doGetJSON(String url, String proxyHost, Integer proxyPort, - Map headers, Map params) { - return RetryUtils.exec(() -> { - HttpGet httpGet = null; - CloseableHttpResponse response = null; - try { - // 创建访问的地址 - URIBuilder uriBuilder = new URIBuilder(url); - if (params != null) { - Set> entrySet = params.entrySet(); - for (Map.Entry entry : entrySet) { - uriBuilder.setParameter(entry.getKey(), entry.getValue()); + /** get json */ + public static HttpClientResult doGetJSON( + String url, + String proxyHost, + Integer proxyPort, + Map headers, + Map params) { + return RetryUtils.exec( + () -> { + HttpGet httpGet = null; + CloseableHttpResponse response = null; + try { + // 创建访问的地址 + URIBuilder uriBuilder = new URIBuilder(url); + if (params != null) { + Set> entrySet = params.entrySet(); + for (Map.Entry entry : entrySet) { + uriBuilder.setParameter(entry.getKey(), entry.getValue()); + } + } + + httpGet = new HttpGet(uriBuilder.build()); + setProxy(httpGet, proxyHost, proxyPort); + + // 设置请求头 + packageHeader(headers, httpGet); + httpGet.setHeader("Content-Type", "application/json;charset=UTF-8"); + + response = httpClient.execute(httpGet); + + // 获取返回结果 + HttpClientResult res = getHttpClientResult(response); + + log.info( + "doGetJSON uri:{}, req:{}, resp:{}", + url, + "headers:" + getHeaders(httpGet) + "------params:" + params, + res); + + return res; + } catch (Exception e) { + log.warn( + "doGetJSON error! uri:{}, req:{}", + url, + "headers:" + headers + "------params:" + params, + e); + throw new RuntimeException(e.getMessage()); + } finally { + close(httpGet, response); } - } - - httpGet = new HttpGet(uriBuilder.build()); - setProxy(httpGet, proxyHost, proxyPort); - - // 设置请求头 - packageHeader(headers, httpGet); - httpGet.setHeader("Content-Type", "application/json;charset=UTF-8"); - - response = httpClient.execute(httpGet); - - // 获取返回结果 - HttpClientResult res = getHttpClientResult(response); - - log.info("doGetJSON uri:{}, req:{}, resp:{}", url, - "headers:" + getHeaders(httpGet) + "------params:" + params, res); - - return res; - } catch (Exception e) { - log.warn("doGetJSON error! uri:{}, req:{}", url, "headers:" + headers + "------params:" + params, e); - throw new RuntimeException(e.getMessage()); - } finally { - close(httpGet, response); - } - }); + }); } - private static HttpClientResult getHttpClientResult(CloseableHttpResponse response) throws IOException { + private static HttpClientResult getHttpClientResult(CloseableHttpResponse response) + throws IOException { HttpEntity entity; HttpClientResult res = new HttpClientResult(HttpStatus.SC_INTERNAL_SERVER_ERROR); if (response != null && response.getStatusLine() != null) { @@ -512,77 +564,94 @@ public class HttpClientUtils { * @param fullFilePath * @return */ - public static HttpClientResult doFileUploadBodyParams(String url, Map headers, + public static HttpClientResult doFileUploadBodyParams( + String url, + Map headers, Map bodyParams, String fullFilePath) { return doFileUpload(url, null, null, headers, null, bodyParams, fullFilePath); } - public static HttpClientResult doFileUpload(String url, + public static HttpClientResult doFileUpload( + String url, String proxyHost, Integer proxyPort, Map headers, Map params, Map bodyParams, String fullFilePath) { - return RetryUtils.exec(() -> { - InputStream inputStream = null; - CloseableHttpResponse response = null; - HttpPost httpPost = null; - try { + return RetryUtils.exec( + () -> { + InputStream inputStream = null; + CloseableHttpResponse response = null; + HttpPost httpPost = null; + try { - File uploadFile = new File(fullFilePath); - inputStream = new FileInputStream(uploadFile); + File uploadFile = new File(fullFilePath); + inputStream = new FileInputStream(uploadFile); - httpPost = new HttpPost(url); - setProxy(httpPost, proxyHost, proxyPort); + httpPost = new HttpPost(url); + setProxy(httpPost, proxyHost, proxyPort); - packageHeader(headers, httpPost); + packageHeader(headers, httpPost); - HttpEntity entity = getFileUploadHttpEntity(params, bodyParams, inputStream, uploadFile.getName()); - httpPost.setEntity(entity); + HttpEntity entity = + getFileUploadHttpEntity( + params, bodyParams, inputStream, uploadFile.getName()); + httpPost.setEntity(entity); - response = httpClient.execute(httpPost); - // 执行请求并获得响应结果 - HttpClientResult res = getHttpClientResult(response); - log.info("doFileUpload uri:{}, req:{}, resp:{}", url, - "params:" + params + ", fullFilePath:" + fullFilePath, res); - return res; - } catch (Exception e) { - log.error("doFileUpload error! uri:{}, req:{}", url, - "params:" + params + ", fullFilePath:" + fullFilePath, e); - throw new RuntimeException(e.getMessage()); - } finally { - try { - if (null != inputStream) { - inputStream.close(); + response = httpClient.execute(httpPost); + // 执行请求并获得响应结果 + HttpClientResult res = getHttpClientResult(response); + log.info( + "doFileUpload uri:{}, req:{}, resp:{}", + url, + "params:" + params + ", fullFilePath:" + fullFilePath, + res); + return res; + } catch (Exception e) { + log.error( + "doFileUpload error! uri:{}, req:{}", + url, + "params:" + params + ", fullFilePath:" + fullFilePath, + e); + throw new RuntimeException(e.getMessage()); + } finally { + try { + if (null != inputStream) { + inputStream.close(); + } + // 释放资源 + close(httpPost, response); + } catch (IOException e) { + log.error("HttpClientUtils release error!", e); + } } - // 释放资源 - close(httpPost, response); - } catch (IOException e) { - log.error("HttpClientUtils release error!", e); - } - } - }); + }); } - private static HttpEntity getFileUploadHttpEntity(Map params, + private static HttpEntity getFileUploadHttpEntity( + Map params, Map bodyParams, InputStream inputStream, - String fileName) throws UnsupportedEncodingException { + String fileName) + throws UnsupportedEncodingException { MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); - builder.addBinaryBody("file", inputStream, ContentType.create("multipart/form-data"), fileName); + builder.addBinaryBody( + "file", inputStream, ContentType.create("multipart/form-data"), fileName); if (!CollectionUtils.isEmpty(bodyParams)) { for (String bodyParamsKey : bodyParams.keySet()) { builder.addTextBody(bodyParamsKey, bodyParams.get(bodyParamsKey)); } } - //构建请求参数 普通表单项 + // 构建请求参数 普通表单项 if (!CollectionUtils.isEmpty(params)) { for (Map.Entry entry : params.entrySet()) { - builder.addPart(entry.getKey(), new StringBody(entry.getValue(), ContentType.MULTIPART_FORM_DATA)); + builder.addPart( + entry.getKey(), + new StringBody(entry.getValue(), ContentType.MULTIPART_FORM_DATA)); } } @@ -599,36 +668,43 @@ public class HttpClientUtils { * @return */ public static HttpClientResult doDelete(String url, Map headers, String req) { - return RetryUtils.exec(() -> { - HttpDeleteWithBody httpDelete = null; - CloseableHttpResponse response = null; - try { - httpDelete = new HttpDeleteWithBody(url); - // 封装header参数 - packageHeader(headers, httpDelete); - httpDelete.setHeader("Content-Type", "application/json;charset=UTF-8"); - // 封装请求参数 - StringEntity stringEntity = new StringEntity(req, ENCODING); // 解决中文乱码问题 - stringEntity.setContentEncoding("UTF-8"); + return RetryUtils.exec( + () -> { + HttpDeleteWithBody httpDelete = null; + CloseableHttpResponse response = null; + try { + httpDelete = new HttpDeleteWithBody(url); + // 封装header参数 + packageHeader(headers, httpDelete); + httpDelete.setHeader("Content-Type", "application/json;charset=UTF-8"); + // 封装请求参数 + StringEntity stringEntity = new StringEntity(req, ENCODING); // 解决中文乱码问题 + stringEntity.setContentEncoding("UTF-8"); - httpDelete.setEntity(stringEntity); + httpDelete.setEntity(stringEntity); - response = httpClient.execute(httpDelete); + response = httpClient.execute(httpDelete); - HttpClientResult res = getHttpClientResult(response); - log.info("doDeleteJSON uri:{}, req:{}, resp:{}", url, - "headers:" + getHeaders(httpDelete) + "------req:" + req, res); - return res; - } catch (Exception e) { - log.error("doDeleteJSON error! uri:{}, req:{}", url, "headers:" + headers + "------req:" + req, e); - throw new RuntimeException(e.getMessage()); - } finally { - close(httpDelete, response); - } - }); + HttpClientResult res = getHttpClientResult(response); + log.info( + "doDeleteJSON uri:{}, req:{}, resp:{}", + url, + "headers:" + getHeaders(httpDelete) + "------req:" + req, + res); + return res; + } catch (Exception e) { + log.error( + "doDeleteJSON error! uri:{}, req:{}", + url, + "headers:" + headers + "------req:" + req, + e); + throw new RuntimeException(e.getMessage()); + } finally { + close(httpDelete, response); + } + }); } - private static class HttpDeleteWithBody extends HttpEntityEnclosingRequestBase { public static final String METHOD_NAME = "DELETE"; @@ -654,30 +730,37 @@ public class HttpClientUtils { } public static HttpClientResult doPutJson(String url, Map headers, String req) { - return RetryUtils.exec(() -> { - HttpPut httpPut = null; - CloseableHttpResponse response = null; - try { - httpPut = new HttpPut(url); - // 封装header参数 - packageHeader(headers, httpPut); - httpPut.setHeader("Content-Type", "application/json;charset=UTF-8"); - // 封装请求参数 - StringEntity stringEntity = new StringEntity(req, ENCODING); // 解决中文乱码问题 - stringEntity.setContentEncoding("UTF-8"); - httpPut.setEntity(stringEntity); - response = httpClient.execute(httpPut); - HttpClientResult res = getHttpClientResult(response); - log.info("doPutJSON uri:{}, req:{}, resp:{}", url, - "headers:" + getHeaders(httpPut) + "------req:" + req, res); - return res; - } catch (Exception e) { - log.error("doPutJSON error! uri:{}, req:{}", url, "headers:" + headers + "------req:" + req, e); - throw new RuntimeException(e.getMessage()); - } finally { - close(httpPut, response); - } - }); + return RetryUtils.exec( + () -> { + HttpPut httpPut = null; + CloseableHttpResponse response = null; + try { + httpPut = new HttpPut(url); + // 封装header参数 + packageHeader(headers, httpPut); + httpPut.setHeader("Content-Type", "application/json;charset=UTF-8"); + // 封装请求参数 + StringEntity stringEntity = new StringEntity(req, ENCODING); // 解决中文乱码问题 + stringEntity.setContentEncoding("UTF-8"); + httpPut.setEntity(stringEntity); + response = httpClient.execute(httpPut); + HttpClientResult res = getHttpClientResult(response); + log.info( + "doPutJSON uri:{}, req:{}, resp:{}", + url, + "headers:" + getHeaders(httpPut) + "------req:" + req, + res); + return res; + } catch (Exception e) { + log.error( + "doPutJSON error! uri:{}, req:{}", + url, + "headers:" + headers + "------req:" + req, + e); + throw new RuntimeException(e.getMessage()); + } finally { + close(httpPut, response); + } + }); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java index 9090f00b6..93516c92a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java @@ -13,6 +13,10 @@ import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.databind.type.MapType; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.nio.charset.Charset; import java.security.InvalidParameterException; @@ -20,29 +24,24 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - @Slf4j public class JsonUtil { public static final JsonUtil INSTANCE = new JsonUtil(); - @Getter - private final ObjectMapper objectMapper = new ObjectMapper(); + @Getter private final ObjectMapper objectMapper = new ObjectMapper(); public JsonUtil() { // 当属性为null时不参与序列化 objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - //允许使用未带引号的字段名 + // 允许使用未带引号的字段名 objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - //忽略未知enum字段，置为null + // 忽略未知enum字段，置为null objectMapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true); - //反序列化忽略未知字段 + // 反序列化忽略未知字段 objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - //允许使用单引号 + // 允许使用单引号 objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); // 遇到空对象不抛异常 objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); @@ -298,7 +297,8 @@ public class JsonUtil { try { notNull(wrapperClass, "wrapperClass is null"); notNull(typeClass, "typeClass is null"); - JavaType type = objectMapper.getTypeFactory().constructParametricType(wrapperClass, typeClass); + JavaType type = + objectMapper.getTypeFactory().constructParametricType(wrapperClass, typeClass); return objectMapper.readValue(json, type); } catch (Exception e) { throw new JsonException(e); @@ -354,7 +354,8 @@ public class JsonUtil { } try { notNull(clazz, "class is null"); - JavaType type = objectMapper.getTypeFactory().constructCollectionType(List.class, clazz); + JavaType type = + objectMapper.getTypeFactory().constructCollectionType(List.class, clazz); return objectMapper.readValue(json, type); } catch (Exception e) { throw new JsonException(e); @@ -400,33 +401,29 @@ public class JsonUtil { notNull(keyClass, "key class is null"); notNull(valueClass, "value class is null"); JavaType type = - objectMapper.getTypeFactory().constructParametricType(Map.class, keyClass, valueClass); + objectMapper + .getTypeFactory() + .constructParametricType(Map.class, keyClass, valueClass); return objectMapper.readValue(json, type); } catch (Exception e) { throw new JsonException(e); } } - /** - * 对象转换成Map - */ + /** 对象转换成Map */ public Map asobjectToMap(Object obj) { if (obj == null) { return null; } - return objectMapper.convertValue(obj, new TypeReference>() { - }); + return objectMapper.convertValue(obj, new TypeReference>() {}); } - /** - * 对象转换成Map - */ + /** 对象转换成Map */ public Map asObjectToMapString(Object obj) { if (obj == null) { return null; } - return objectMapper.convertValue(obj, new TypeReference>() { - }); + return objectMapper.convertValue(obj, new TypeReference>() {}); } /** @@ -455,7 +452,8 @@ public class JsonUtil { try { final TypeFactory typeFactory = getObjectMapper().getTypeFactory(); final MapType mapType = typeFactory.constructMapType(Map.class, String.class, clazz); - final CollectionType collectionType = typeFactory.constructCollectionType(List.class, mapType); + final CollectionType collectionType = + typeFactory.constructCollectionType(List.class, mapType); return getObjectMapper().readValue(json, collectionType); } catch (Exception e) { log.error("json:{} to listMap error:", json, e); @@ -505,7 +503,9 @@ public class JsonUtil { } try { JsonNode jsonNode = readTree(string); - return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode); + return objectMapper + .writerWithDefaultPrettyPrinter() + .writeValueAsString(jsonNode); } catch (Exception e) { return string; } @@ -554,8 +554,8 @@ public class JsonUtil { } } - /*** - * 判断字符串是否合格的json格式 + /** + * * 判断字符串是否合格的json格式 * * @param json * @return @@ -598,15 +598,12 @@ public class JsonUtil { } } - /** - * json序列化,或者反序列化发生的异常 - */ + /** json序列化,或者反序列化发生的异常 */ public static class JsonException extends RuntimeException { private static final long serialVersionUID = 1L; - private JsonException() { - } + private JsonException() {} private JsonException(String message) { super(message); @@ -620,7 +617,10 @@ public class JsonUtil { super(cause); } - private JsonException(String message, Throwable cause, boolean enableSuppression, + private JsonException( + String message, + Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/MD5Util.java b/common/src/main/java/com/tencent/supersonic/common/util/MD5Util.java index 2dd3c4fd3..eec0d8558 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/MD5Util.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/MD5Util.java @@ -1,14 +1,14 @@ package com.tencent.supersonic.common.util; +import lombok.extern.slf4j.Slf4j; + import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.Base64; -import lombok.extern.slf4j.Slf4j; @Slf4j public class MD5Util { - public static final int BIT16 = 16; public static final int BIT32 = 32; public static final int BIT64 = 64; @@ -69,4 +69,4 @@ public class MD5Util { } return buffer.toString(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/PageUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/PageUtils.java index 5d86a382c..3d8978045 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/PageUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/PageUtils.java @@ -5,12 +5,10 @@ import com.github.pagehelper.PageInfo; public class PageUtils { - /** - * convert PageInfo Po to Vo - */ + /** convert PageInfo Po to Vo */ public static PageInfo pageInfo2PageInfoVo(PageInfo

pageInfoPo) { Page page = new Page<>(pageInfoPo.getPageNum(), pageInfoPo.getPageSize()); page.setTotal(pageInfoPo.getTotal()); return new PageInfo<>(page); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/RetryUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/RetryUtils.java index 03dc0392e..c170458b8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/RetryUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/RetryUtils.java @@ -1,14 +1,12 @@ package com.tencent.supersonic.common.util; - -import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; +import java.util.function.Supplier; @Slf4j public class RetryUtils { - private static final int RETRY_NUM = 3; public static T exec(Supplier supplier) { @@ -38,5 +36,4 @@ public class RetryUtils { return result; } - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/S2ThreadContext.java b/common/src/main/java/com/tencent/supersonic/common/util/S2ThreadContext.java index b3ba60dda..91db635a2 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/S2ThreadContext.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/S2ThreadContext.java @@ -4,7 +4,8 @@ import com.alibaba.ttl.TransmittableThreadLocal; public class S2ThreadContext { - private static final TransmittableThreadLocal context = new TransmittableThreadLocal<>(); + private static final TransmittableThreadLocal context = + new TransmittableThreadLocal<>(); public ThreadContext get() { return context.get(); @@ -17,4 +18,4 @@ public class S2ThreadContext { public void remove() { context.remove(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java index 487c4c9b0..87cb07e67 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java @@ -12,8 +12,8 @@ public class SignatureUtils { return DigestUtils.sha1Hex(psw); } - public static Pair isValidSignature(String appKey, String appSecret, - long timestamp, String signatureToCheck) { + public static Pair isValidSignature( + String appKey, String appSecret, long timestamp, String signatureToCheck) { long currentTimeMillis = System.currentTimeMillis(); if (currentTimeMillis < timestamp) { @@ -34,16 +34,16 @@ public class SignatureUtils { } public static void main(String[] args) throws InterruptedException { - //生成的密钥 + // 生成的密钥 String appSecret = "38f2857c-d9ee-4c3a-bcc2-2cdb62fda5aa"; long timestamp = 1706504908126L; System.out.println("timeStamp:" + timestamp); - //生成的签名 + // 生成的签名 String serverSignature = generateSignature(appSecret, timestamp); System.out.println("Server Signature: " + serverSignature); - //用户需要的入参 - Pair isValid = isValidSignature("1", appSecret, timestamp, serverSignature); + // 用户需要的入参 + Pair isValid = + isValidSignature("1", appSecret, timestamp, serverSignature); System.out.println("Is Signature Valid? " + isValid.first); } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/SqlFilterUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/SqlFilterUtils.java index 214fdf5ba..1d0b6aede 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/SqlFilterUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/SqlFilterUtils.java @@ -20,9 +20,6 @@ import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START; import static com.tencent.supersonic.common.pojo.Constants.SPACE; import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR; -; - - @Component @Slf4j public class SqlFilterUtils { @@ -63,11 +60,12 @@ public class SqlFilterUtils { if (!CollectionUtils.isEmpty(filters)) { filters.stream() - .forEach(filter -> { - if (StringUtils.isNotEmpty(dealFilter(filter, isBizName))) { - joiner.add(SPACE + dealFilter(filter, isBizName) + SPACE); - } - }); + .forEach( + filter -> { + if (StringUtils.isNotEmpty(dealFilter(filter, isBizName))) { + joiner.add(SPACE + dealFilter(filter, isBizName) + SPACE); + } + }); log.debug("getWhereClause, where sql : {}", joiner); return joiner.toString(); } @@ -162,16 +160,20 @@ public class SqlFilterUtils { throw new RuntimeException("criterion.getValue() can not be null"); } StringBuilder whereClause = new StringBuilder(); - whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE); + whereClause.append( + criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE); String value = criterion.getValue().toString(); if (criterion.isNeedApostrophe() && !Pattern.matches(pattern, value)) { // like click => 'like%' - whereClause.append(Constants.APOSTROPHE + value + Constants.PERCENT_SIGN + Constants.APOSTROPHE); + whereClause.append( + Constants.APOSTROPHE + value + Constants.PERCENT_SIGN + Constants.APOSTROPHE); } else { // like 'click' => 'like%' - whereClause.append(Constants.APOSTROPHE + value.replaceAll(Constants.APOSTROPHE, Constants.PERCENT_SIGN) - + Constants.APOSTROPHE); + whereClause.append( + Constants.APOSTROPHE + + value.replaceAll(Constants.APOSTROPHE, Constants.PERCENT_SIGN) + + Constants.APOSTROPHE); } return whereClause.toString(); } @@ -182,7 +184,8 @@ public class SqlFilterUtils { } StringBuilder whereClause = new StringBuilder(); - whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE); + whereClause.append( + criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE); List values = (List) criterion.getValue(); whereClause.append(PARENTHESES_START); StringJoiner joiner = new StringJoiner(","); @@ -206,12 +209,19 @@ public class SqlFilterUtils { } if (criterion.isNeedApostrophe()) { - return String.format("(%s >= %s and %s <= %s)", criterion.getColumn(), + return String.format( + "(%s >= %s and %s <= %s)", + criterion.getColumn(), valueApostropheLogic(values.get(0).toString()), - criterion.getColumn(), valueApostropheLogic(values.get(1).toString())); + criterion.getColumn(), + valueApostropheLogic(values.get(1).toString())); } - return String.format("(%s >= %s and %s <= %s)", criterion.getColumn(), values.get(0).toString(), - criterion.getColumn(), values.get(1).toString()); + return String.format( + "(%s >= %s and %s <= %s)", + criterion.getColumn(), + values.get(0).toString(), + criterion.getColumn(), + values.get(1).toString()); } private String singleValueLogic(Criterion criterion) { @@ -219,7 +229,8 @@ public class SqlFilterUtils { throw new RuntimeException("criterion.getValue() can not be null"); } StringBuilder whereClause = new StringBuilder(); - whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE); + whereClause.append( + criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE); String value = criterion.getValue().toString(); if (criterion.isNeedApostrophe()) { value = valueApostropheLogic(value); @@ -247,7 +258,10 @@ public class SqlFilterUtils { if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) { throw new RuntimeException("criterion.getValue() can not be null"); } - return PARENTHESES_START + SPACE + criterion.getValue().toString() + SPACE + PARENTHESES_END; + return PARENTHESES_START + + SPACE + + criterion.getValue().toString() + + SPACE + + PARENTHESES_END; } - } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java index c7b3f3f08..36ce4cce6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.common.util; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; - @Slf4j public class StringUtil { @@ -28,7 +27,8 @@ public class StringUtil { /** * @param v1 * @param v2 - * @return value 0 if v1 equal to v2; less than 0 if v1 is less than v2; greater than 0 if v1 is greater than v2 + * @return value 0 if v1 equal to v2; less than 0 if v1 is less than v2; greater than 0 if v1 is + * greater than v2 */ public static int compareVersion(String v1, String v2) { String[] v1s = v1.split("\\."); @@ -48,4 +48,4 @@ public class StringUtil { public static String replaceBackticks(String sql) { return sql.replaceAll("`", ""); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java b/common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java index 7399e787a..fc38dc4cc 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.common.util; - import lombok.Builder; import lombok.Data; import lombok.ToString; + import java.util.Map; @Builder @@ -18,5 +18,4 @@ public class ThreadContext { private String token; private Map extendInfo; - -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java index 81bf74313..acd4ddf40 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java @@ -12,7 +12,8 @@ public class ThreadMdcUtil { } } - public static Callable wrap(final Callable callable, final Map context) { + public static Callable wrap( + final Callable callable, final Map context) { return () -> { if (context == null) { MDC.clear(); @@ -35,7 +36,7 @@ public class ThreadMdcUtil { } else { MDC.setContextMap(context); } - //设置traceId + // 设置traceId setTraceIdIfAbsent(); try { runnable.run(); @@ -45,5 +46,3 @@ public class ThreadMdcUtil { }; } } - - diff --git a/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java index 025997bc2..2e4d2a175 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java @@ -20,7 +20,6 @@ public class TraceIdUtil { public static void remove() { MDC.remove(TRACE_ID); - } public static void clear() { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/YamlUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/YamlUtils.java index e4354799f..cb79c3b3d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/YamlUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/YamlUtils.java @@ -9,10 +9,11 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import lombok.extern.slf4j.Slf4j; + import java.io.IOException; import java.util.LinkedHashMap; import java.util.List; -import lombok.extern.slf4j.Slf4j; @Slf4j public class YamlUtils { @@ -46,7 +47,8 @@ public class YamlUtils { YAMLMapper mapper = new YAMLMapper(); mapper.findAndRegisterModules(); mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - mapper.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES).disable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); + mapper.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .disable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); try { String yaml = mapper.writeValueAsString(object); return yaml.replaceAll("\"True\"", "true") @@ -64,15 +66,13 @@ public class YamlUtils { if (object instanceof List) { return toYaml(JSONObject.parseObject(jsonStr, List.class, Feature.OrderedField)); } else { - return toYaml(JSONObject.parseObject(jsonStr, LinkedHashMap.class, Feature.OrderedField)); + return toYaml( + JSONObject.parseObject(jsonStr, LinkedHashMap.class, Feature.OrderedField)); } - - } /** - * （此方法非必要） - * json 2 yaml + * （此方法非必要） json 2 yaml * * @param jsonStr json * @return yaml @@ -82,5 +82,4 @@ public class YamlUtils { JsonNode jsonNode = new ObjectMapper().readTree(jsonStr); return new YAMLMapper().writeValueAsString(jsonNode); } - } diff --git a/common/src/main/java/dev/langchain4j/chroma/spring/ChromaAutoConfig.java b/common/src/main/java/dev/langchain4j/chroma/spring/ChromaAutoConfig.java index fc68dd575..319255729 100644 --- a/common/src/main/java/dev/langchain4j/chroma/spring/ChromaAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/chroma/spring/ChromaAutoConfig.java @@ -1,6 +1,5 @@ package dev.langchain4j.chroma.spring; - import dev.langchain4j.store.embedding.EmbeddingStoreFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -18,4 +17,4 @@ public class ChromaAutoConfig { EmbeddingStoreFactory chromaChatModel(Properties properties) { return new ChromaEmbeddingStoreFactory(properties.getEmbeddingStore()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/chroma/spring/ChromaEmbeddingStoreFactory.java b/common/src/main/java/dev/langchain4j/chroma/spring/ChromaEmbeddingStoreFactory.java index dca01b3af..dd980881c 100644 --- a/common/src/main/java/dev/langchain4j/chroma/spring/ChromaEmbeddingStoreFactory.java +++ b/common/src/main/java/dev/langchain4j/chroma/spring/ChromaEmbeddingStoreFactory.java @@ -31,10 +31,11 @@ public class ChromaEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { .build(); } - private static EmbeddingStoreProperties createPropertiesFromConfig(EmbeddingStoreConfig storeConfig) { + private static EmbeddingStoreProperties createPropertiesFromConfig( + EmbeddingStoreConfig storeConfig) { EmbeddingStoreProperties embeddingStore = new EmbeddingStoreProperties(); BeanUtils.copyProperties(storeConfig, embeddingStore); embeddingStore.setTimeout(Duration.ofSeconds(storeConfig.getTimeOut())); return embeddingStore; } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/chroma/spring/EmbeddingStoreProperties.java b/common/src/main/java/dev/langchain4j/chroma/spring/EmbeddingStoreProperties.java index 72e7f5d9e..982482de5 100644 --- a/common/src/main/java/dev/langchain4j/chroma/spring/EmbeddingStoreProperties.java +++ b/common/src/main/java/dev/langchain4j/chroma/spring/EmbeddingStoreProperties.java @@ -13,4 +13,4 @@ public class EmbeddingStoreProperties { private String baseUrl; private Duration timeout; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/chroma/spring/Properties.java b/common/src/main/java/dev/langchain4j/chroma/spring/Properties.java index 7823ca677..26141ff19 100644 --- a/common/src/main/java/dev/langchain4j/chroma/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/chroma/spring/Properties.java @@ -12,6 +12,5 @@ public class Properties { static final String PREFIX = "langchain4j.chroma"; - @NestedConfigurationProperty - EmbeddingStoreProperties embeddingStore; -} \ No newline at end of file + @NestedConfigurationProperty EmbeddingStoreProperties embeddingStore; +} diff --git a/common/src/main/java/dev/langchain4j/dashscope/spring/ChatModelProperties.java b/common/src/main/java/dev/langchain4j/dashscope/spring/ChatModelProperties.java index 7a407cb8f..5cfc33d62 100644 --- a/common/src/main/java/dev/langchain4j/dashscope/spring/ChatModelProperties.java +++ b/common/src/main/java/dev/langchain4j/dashscope/spring/ChatModelProperties.java @@ -1,9 +1,10 @@ package dev.langchain4j.dashscope.spring; -import java.util.List; import lombok.Getter; import lombok.Setter; +import java.util.List; + @Getter @Setter class ChatModelProperties { @@ -19,4 +20,4 @@ class ChatModelProperties { Float temperature; List stops; Integer maxTokens; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/dashscope/spring/DashscopeAutoConfig.java b/common/src/main/java/dev/langchain4j/dashscope/spring/DashscopeAutoConfig.java index 61cf9106f..db915d59d 100644 --- a/common/src/main/java/dev/langchain4j/dashscope/spring/DashscopeAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/dashscope/spring/DashscopeAutoConfig.java @@ -1,8 +1,5 @@ package dev.langchain4j.dashscope.spring; - -import static dev.langchain4j.dashscope.spring.Properties.PREFIX; - import dev.langchain4j.model.dashscope.QwenChatModel; import dev.langchain4j.model.dashscope.QwenEmbeddingModel; import dev.langchain4j.model.dashscope.QwenLanguageModel; @@ -13,6 +10,8 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static dev.langchain4j.dashscope.spring.Properties.PREFIX; + @Configuration @EnableConfigurationProperties(Properties.class) public class DashscopeAutoConfig { @@ -102,4 +101,4 @@ public class DashscopeAutoConfig { .modelName(embeddingModelProperties.getModelName()) .build(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/dashscope/spring/EmbeddingModelProperties.java b/common/src/main/java/dev/langchain4j/dashscope/spring/EmbeddingModelProperties.java index 27b09525f..78c0b0b1e 100644 --- a/common/src/main/java/dev/langchain4j/dashscope/spring/EmbeddingModelProperties.java +++ b/common/src/main/java/dev/langchain4j/dashscope/spring/EmbeddingModelProperties.java @@ -9,4 +9,4 @@ class EmbeddingModelProperties { private String apiKey; private String modelName; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/dashscope/spring/Properties.java b/common/src/main/java/dev/langchain4j/dashscope/spring/Properties.java index 2b8215626..53852c433 100644 --- a/common/src/main/java/dev/langchain4j/dashscope/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/dashscope/spring/Properties.java @@ -12,18 +12,13 @@ public class Properties { static final String PREFIX = "langchain4j.dashscope"; - @NestedConfigurationProperty - ChatModelProperties chatModel; + @NestedConfigurationProperty ChatModelProperties chatModel; - @NestedConfigurationProperty - ChatModelProperties streamingChatModel; + @NestedConfigurationProperty ChatModelProperties streamingChatModel; - @NestedConfigurationProperty - ChatModelProperties languageModel; + @NestedConfigurationProperty ChatModelProperties languageModel; - @NestedConfigurationProperty - ChatModelProperties streamingLanguageModel; + @NestedConfigurationProperty ChatModelProperties streamingLanguageModel; - @NestedConfigurationProperty - EmbeddingModelProperties embeddingModel; -} \ No newline at end of file + @NestedConfigurationProperty EmbeddingModelProperties embeddingModel; +} diff --git a/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingModelProperties.java b/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingModelProperties.java index 3e9c529a2..01e7733fe 100644 --- a/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingModelProperties.java +++ b/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingModelProperties.java @@ -10,4 +10,4 @@ class EmbeddingModelProperties { private String modelName; private String modelPath; private String vocabularyPath; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingStoreProperties.java b/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingStoreProperties.java index e1348e69c..1cb07bd2f 100644 --- a/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingStoreProperties.java +++ b/common/src/main/java/dev/langchain4j/inmemory/spring/EmbeddingStoreProperties.java @@ -8,4 +8,4 @@ import lombok.Setter; public class EmbeddingStoreProperties { private String persistPath; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java b/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java index 03d1c0ebc..5b19c05e4 100644 --- a/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java @@ -1,6 +1,5 @@ package dev.langchain4j.inmemory.spring; - import dev.langchain4j.model.embedding.AllMiniLmL6V2QuantizedEmbeddingModel; import dev.langchain4j.model.embedding.BgeSmallZhEmbeddingModel; import dev.langchain4j.model.embedding.EmbeddingModel; @@ -42,4 +41,4 @@ public class InMemoryAutoConfig { } return new BgeSmallZhEmbeddingModel(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryEmbeddingStoreFactory.java b/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryEmbeddingStoreFactory.java index 20aa64838..56c35de95 100644 --- a/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryEmbeddingStoreFactory.java +++ b/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryEmbeddingStoreFactory.java @@ -33,7 +33,8 @@ public class InMemoryEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { this.embeddingStore = embeddingStore; } - private static EmbeddingStoreProperties createPropertiesFromConfig(EmbeddingStoreConfig storeConfig) { + private static EmbeddingStoreProperties createPropertiesFromConfig( + EmbeddingStoreConfig storeConfig) { EmbeddingStoreProperties embeddingStore = new EmbeddingStoreProperties(); BeanUtils.copyProperties(storeConfig, embeddingStore); return embeddingStore; @@ -56,7 +57,8 @@ public class InMemoryEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { InMemoryEmbeddingStore embeddingStore = null; try { EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); - if (Files.exists(filePath) && !collectionName.equals(embeddingConfig.getMetaCollectionName()) + if (Files.exists(filePath) + && !collectionName.equals(embeddingConfig.getMetaCollectionName()) && !collectionName.equals(embeddingConfig.getText2sqlCollectionName())) { embeddingStore = InMemoryEmbeddingStore.fromFile(filePath); embeddingStore.entries = new CopyOnWriteArraySet<>(embeddingStore.entries); @@ -72,7 +74,8 @@ public class InMemoryEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { if (MapUtils.isEmpty(super.collectionNameToStore)) { return; } - for (Map.Entry> entry : collectionNameToStore.entrySet()) { + for (Map.Entry> entry : + collectionNameToStore.entrySet()) { Path filePath = getPersistPath(entry.getKey()); if (Objects.isNull(filePath)) { continue; @@ -101,5 +104,4 @@ public class InMemoryEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { } return Paths.get(persistPath, persistFile); } - -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/inmemory/spring/Properties.java b/common/src/main/java/dev/langchain4j/inmemory/spring/Properties.java index 47f2eb298..281496c8e 100644 --- a/common/src/main/java/dev/langchain4j/inmemory/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/inmemory/spring/Properties.java @@ -12,9 +12,7 @@ public class Properties { static final String PREFIX = "langchain4j.in-memory"; - @NestedConfigurationProperty - EmbeddingStoreProperties embeddingStore; + @NestedConfigurationProperty EmbeddingStoreProperties embeddingStore; - @NestedConfigurationProperty - EmbeddingModelProperties embeddingModel; -} \ No newline at end of file + @NestedConfigurationProperty EmbeddingModelProperties embeddingModel; +} diff --git a/common/src/main/java/dev/langchain4j/localai/spring/ChatModelProperties.java b/common/src/main/java/dev/langchain4j/localai/spring/ChatModelProperties.java index bdb25daaf..83302d792 100644 --- a/common/src/main/java/dev/langchain4j/localai/spring/ChatModelProperties.java +++ b/common/src/main/java/dev/langchain4j/localai/spring/ChatModelProperties.java @@ -18,4 +18,4 @@ class ChatModelProperties { private Double penaltyScore; private Boolean logRequests; private Boolean logResponses; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/localai/spring/EmbeddingModelProperties.java b/common/src/main/java/dev/langchain4j/localai/spring/EmbeddingModelProperties.java index 932f3fa47..60271a209 100644 --- a/common/src/main/java/dev/langchain4j/localai/spring/EmbeddingModelProperties.java +++ b/common/src/main/java/dev/langchain4j/localai/spring/EmbeddingModelProperties.java @@ -15,5 +15,4 @@ class EmbeddingModelProperties { private String user; private Boolean logRequests; private Boolean logResponses; - -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/localai/spring/LanguageModelProperties.java b/common/src/main/java/dev/langchain4j/localai/spring/LanguageModelProperties.java index c9bb80a9a..4634672f5 100644 --- a/common/src/main/java/dev/langchain4j/localai/spring/LanguageModelProperties.java +++ b/common/src/main/java/dev/langchain4j/localai/spring/LanguageModelProperties.java @@ -18,5 +18,4 @@ class LanguageModelProperties { private Double penaltyScore; private Boolean logRequests; private Boolean logResponses; - -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/localai/spring/LocalAiAutoConfig.java b/common/src/main/java/dev/langchain4j/localai/spring/LocalAiAutoConfig.java index f3fb521cf..817451b1a 100644 --- a/common/src/main/java/dev/langchain4j/localai/spring/LocalAiAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/localai/spring/LocalAiAutoConfig.java @@ -86,4 +86,4 @@ public class LocalAiAutoConfig { .logResponses(embeddingModelProperties.getLogResponses()) .build(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/localai/spring/Properties.java b/common/src/main/java/dev/langchain4j/localai/spring/Properties.java index 61ae589b7..e8b399ead 100644 --- a/common/src/main/java/dev/langchain4j/localai/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/localai/spring/Properties.java @@ -12,18 +12,13 @@ public class Properties { static final String PREFIX = "langchain4j.local-ai"; - @NestedConfigurationProperty - ChatModelProperties chatModel; + @NestedConfigurationProperty ChatModelProperties chatModel; - @NestedConfigurationProperty - ChatModelProperties streamingChatModel; + @NestedConfigurationProperty ChatModelProperties streamingChatModel; - @NestedConfigurationProperty - LanguageModelProperties languageModel; + @NestedConfigurationProperty LanguageModelProperties languageModel; - @NestedConfigurationProperty - LanguageModelProperties streamingLanguageModel; + @NestedConfigurationProperty LanguageModelProperties streamingLanguageModel; - @NestedConfigurationProperty - EmbeddingModelProperties embeddingModel; + @NestedConfigurationProperty EmbeddingModelProperties embeddingModel; } diff --git a/common/src/main/java/dev/langchain4j/milvus/spring/EmbeddingStoreProperties.java b/common/src/main/java/dev/langchain4j/milvus/spring/EmbeddingStoreProperties.java index 0dab5f45a..52530a7f0 100644 --- a/common/src/main/java/dev/langchain4j/milvus/spring/EmbeddingStoreProperties.java +++ b/common/src/main/java/dev/langchain4j/milvus/spring/EmbeddingStoreProperties.java @@ -24,4 +24,4 @@ class EmbeddingStoreProperties { private Boolean retrieveEmbeddingsOnSearch; private String databaseName; private Boolean autoFlushOnInsert; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/milvus/spring/MilvusAutoConfig.java b/common/src/main/java/dev/langchain4j/milvus/spring/MilvusAutoConfig.java index 47ff35ba6..81d238a50 100644 --- a/common/src/main/java/dev/langchain4j/milvus/spring/MilvusAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/milvus/spring/MilvusAutoConfig.java @@ -17,4 +17,4 @@ public class MilvusAutoConfig { EmbeddingStoreFactory milvusChatModel(Properties properties) { return new MilvusEmbeddingStoreFactory(properties.getEmbeddingStore()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/milvus/spring/MilvusEmbeddingStoreFactory.java b/common/src/main/java/dev/langchain4j/milvus/spring/MilvusEmbeddingStoreFactory.java index 3e3aac552..1f2d15f0f 100644 --- a/common/src/main/java/dev/langchain4j/milvus/spring/MilvusEmbeddingStoreFactory.java +++ b/common/src/main/java/dev/langchain4j/milvus/spring/MilvusEmbeddingStoreFactory.java @@ -18,7 +18,8 @@ public class MilvusEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { this.storeProperties = storeProperties; } - private static EmbeddingStoreProperties createPropertiesFromConfig(EmbeddingStoreConfig storeConfig) { + private static EmbeddingStoreProperties createPropertiesFromConfig( + EmbeddingStoreConfig storeConfig) { EmbeddingStoreProperties embeddingStore = new EmbeddingStoreProperties(); BeanUtils.copyProperties(storeConfig, embeddingStore); embeddingStore.setUri(storeConfig.getBaseUrl()); @@ -45,4 +46,4 @@ public class MilvusEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { .databaseName(storeProperties.getDatabaseName()) .build(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/milvus/spring/Properties.java b/common/src/main/java/dev/langchain4j/milvus/spring/Properties.java index 5c54b08d5..36ea54e24 100644 --- a/common/src/main/java/dev/langchain4j/milvus/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/milvus/spring/Properties.java @@ -12,6 +12,5 @@ public class Properties { static final String PREFIX = "langchain4j.milvus"; - @NestedConfigurationProperty - EmbeddingStoreProperties embeddingStore; -} \ No newline at end of file + @NestedConfigurationProperty EmbeddingStoreProperties embeddingStore; +} diff --git a/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java b/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java index 87bec18ca..a1ba91439 100644 --- a/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java +++ b/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java @@ -11,12 +11,13 @@ import java.nio.file.Paths; import java.util.Objects; /** - * An embedding model that runs within your Java application's process. - * Any BERT-based model (e.g., from HuggingFace) can be used, as long as it is in ONNX format. - * Information on how to convert models into ONNX format can be found here. - * Many models already converted to ONNX format are available here. - * Copy from dev.langchain4j.model.embedding.OnnxEmbeddingModel. + * Many models already converted to ONNX format are available here. Copy from + * dev.langchain4j.model.embedding.OnnxEmbeddingModel. */ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel { private static volatile OnnxBertBiEncoder cachedModel; @@ -27,7 +28,9 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel { if (shouldReloadModel(pathToModel, vocabularyPath)) { synchronized (S2OnnxEmbeddingModel.class) { if (shouldReloadModel(pathToModel, vocabularyPath)) { - URL resource = AbstractInProcessEmbeddingModel.class.getResource("/bert-vocabulary-en.txt"); + URL resource = + AbstractInProcessEmbeddingModel.class.getResource( + "/bert-vocabulary-en.txt"); if (StringUtils.isNotBlank(vocabularyPath)) { try { resource = Paths.get(vocabularyPath).toUri().toURL(); @@ -53,19 +56,17 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel { } private static boolean shouldReloadModel(String pathToModel, String vocabularyPath) { - return cachedModel == null || !Objects.equals(cachedModelPath, pathToModel) + return cachedModel == null + || !Objects.equals(cachedModelPath, pathToModel) || !Objects.equals(cachedVocabularyPath, vocabularyPath); } static OnnxBertBiEncoder loadFromFileSystem(Path pathToModel, URL vocabularyFile) { try { return new OnnxBertBiEncoder( - Files.newInputStream(pathToModel), - vocabularyFile, - PoolingMode.MEAN - ); + Files.newInputStream(pathToModel), vocabularyFile, PoolingMode.MEAN); } catch (IOException e) { throw new RuntimeException(e); } } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/model/openai/OpenAiChatModel.java b/common/src/main/java/dev/langchain4j/model/openai/OpenAiChatModel.java index 70c76e472..65ce04a75 100644 --- a/common/src/main/java/dev/langchain4j/model/openai/OpenAiChatModel.java +++ b/common/src/main/java/dev/langchain4j/model/openai/OpenAiChatModel.java @@ -44,8 +44,9 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; /** - * Represents an OpenAI language model with a chat completion interface, such as gpt-3.5-turbo and gpt-4. - * You can find description of parameters here. + * Represents an OpenAI language model with a chat completion interface, such as gpt-3.5-turbo and + * gpt-4. You can find description of parameters here. */ @Slf4j public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { @@ -67,31 +68,33 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { private final Integer maxRetries; private final Tokenizer tokenizer; - private final List> listeners; + private final List> + listeners; @Builder - public OpenAiChatModel(String baseUrl, - String apiKey, - String organizationId, - String modelName, - Double temperature, - Double topP, - List stop, - Integer maxTokens, - Double presencePenalty, - Double frequencyPenalty, - Map logitBias, - String responseFormat, - Integer seed, - String user, - Duration timeout, - Integer maxRetries, - Proxy proxy, - Boolean logRequests, - Boolean logResponses, - Tokenizer tokenizer, - Map customHeaders, - List> listeners) { + public OpenAiChatModel( + String baseUrl, + String apiKey, + String organizationId, + String modelName, + Double temperature, + Double topP, + List stop, + Integer maxTokens, + Double presencePenalty, + Double frequencyPenalty, + Map logitBias, + String responseFormat, + Integer seed, + String user, + Duration timeout, + Integer maxRetries, + Proxy proxy, + Boolean logRequests, + Boolean logResponses, + Tokenizer tokenizer, + Map customHeaders, + List> listeners) { baseUrl = getOrDefault(baseUrl, OPENAI_URL); if (OPENAI_DEMO_API_KEY.equals(apiKey)) { @@ -101,20 +104,21 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { timeout = getOrDefault(timeout, ofSeconds(60)); - this.client = OpenAiClient.builder() - .openAiApiKey(apiKey) - .baseUrl(baseUrl) - .organizationId(organizationId) - .callTimeout(timeout) - .connectTimeout(timeout) - .readTimeout(timeout) - .writeTimeout(timeout) - .proxy(proxy) - .logRequests(logRequests) - .logResponses(logResponses) - .userAgent(DEFAULT_USER_AGENT) - .customHeaders(customHeaders) - .build(); + this.client = + OpenAiClient.builder() + .openAiApiKey(apiKey) + .baseUrl(baseUrl) + .organizationId(organizationId) + .callTimeout(timeout) + .connectTimeout(timeout) + .readTimeout(timeout) + .writeTimeout(timeout) + .proxy(proxy) + .logRequests(logRequests) + .logResponses(logResponses) + .userAgent(DEFAULT_USER_AGENT) + .customHeaders(customHeaders) + .build(); this.modelName = getOrDefault(modelName, GPT_3_5_TURBO); this.temperature = getOrDefault(temperature, 0.7); this.topP = topP; @@ -141,31 +145,34 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { } @Override - public Response generate(List messages, List toolSpecifications) { + public Response generate( + List messages, List toolSpecifications) { return generate(messages, toolSpecifications, null); } @Override - public Response generate(List messages, ToolSpecification toolSpecification) { + public Response generate( + List messages, ToolSpecification toolSpecification) { return generate(messages, singletonList(toolSpecification), toolSpecification); } - private Response generate(List messages, - List toolSpecifications, - ToolSpecification toolThatMustBeExecuted - ) { - ChatCompletionRequest.Builder requestBuilder = ChatCompletionRequest.builder() - .model(modelName) - .messages(toOpenAiMessages(messages)) - .topP(topP) - .stop(stop) - .maxTokens(maxTokens) - .presencePenalty(presencePenalty) - .frequencyPenalty(frequencyPenalty) - .logitBias(logitBias) - .responseFormat(responseFormat) - .seed(seed) - .user(user); + private Response generate( + List messages, + List toolSpecifications, + ToolSpecification toolThatMustBeExecuted) { + ChatCompletionRequest.Builder requestBuilder = + ChatCompletionRequest.builder() + .model(modelName) + .messages(toOpenAiMessages(messages)) + .topP(topP) + .stop(stop) + .maxTokens(maxTokens) + .presencePenalty(presencePenalty) + .frequencyPenalty(frequencyPenalty) + .logitBias(logitBias) + .responseFormat(responseFormat) + .seed(seed) + .user(user); if (!(baseUrl.contains(ZHIPU))) { requestBuilder.temperature(temperature); } @@ -181,36 +188,37 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { ChatLanguageModelRequest modelListenerRequest = createModelListenerRequest(request, messages, toolSpecifications); - listeners.forEach(listener -> { - try { - listener.onRequest(modelListenerRequest); - } catch (Exception e) { - log.warn("Exception while calling model listener", e); - } - }); + listeners.forEach( + listener -> { + try { + listener.onRequest(modelListenerRequest); + } catch (Exception e) { + log.warn("Exception while calling model listener", e); + } + }); try { ChatCompletionResponse chatCompletionResponse = withRetry(() -> client.chatCompletion(request).execute(), maxRetries); - Response response = Response.from( - aiMessageFrom(chatCompletionResponse), - tokenUsageFrom(chatCompletionResponse.usage()), - finishReasonFrom(chatCompletionResponse.choices().get(0).finishReason()) - ); + Response response = + Response.from( + aiMessageFrom(chatCompletionResponse), + tokenUsageFrom(chatCompletionResponse.usage()), + finishReasonFrom( + chatCompletionResponse.choices().get(0).finishReason())); - ChatLanguageModelResponse modelListenerResponse = createModelListenerResponse( - chatCompletionResponse.id(), - chatCompletionResponse.model(), - response - ); - listeners.forEach(listener -> { - try { - listener.onResponse(modelListenerResponse, modelListenerRequest); - } catch (Exception e) { - log.warn("Exception while calling model listener", e); - } - }); + ChatLanguageModelResponse modelListenerResponse = + createModelListenerResponse( + chatCompletionResponse.id(), chatCompletionResponse.model(), response); + listeners.forEach( + listener -> { + try { + listener.onResponse(modelListenerResponse, modelListenerRequest); + } catch (Exception e) { + log.warn("Exception while calling model listener", e); + } + }); return response; } catch (RuntimeException e) { @@ -222,13 +230,14 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { error = e; } - listeners.forEach(listener -> { - try { - listener.onError(error, null, modelListenerRequest); - } catch (Exception e2) { - log.warn("Exception while calling model listener", e2); - } - }); + listeners.forEach( + listener -> { + try { + listener.onError(error, null, modelListenerRequest); + } catch (Exception e2) { + log.warn("Exception while calling model listener", e2); + } + }); throw e; } } @@ -243,7 +252,8 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { } public static OpenAiChatModelBuilder builder() { - for (OpenAiChatModelBuilderFactory factory : loadFactories(OpenAiChatModelBuilderFactory.class)) { + for (OpenAiChatModelBuilderFactory factory : + loadFactories(OpenAiChatModelBuilderFactory.class)) { return factory.get(); } return new OpenAiChatModelBuilder(); @@ -261,4 +271,4 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator { return this; } } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/model/zhipu/ChatCompletionModel.java b/common/src/main/java/dev/langchain4j/model/zhipu/ChatCompletionModel.java index b0496e819..6ea276723 100644 --- a/common/src/main/java/dev/langchain4j/model/zhipu/ChatCompletionModel.java +++ b/common/src/main/java/dev/langchain4j/model/zhipu/ChatCompletionModel.java @@ -15,4 +15,4 @@ public enum ChatCompletionModel { public String toString() { return this.value; } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/model/zhipu/ZhipuAiChatModel.java b/common/src/main/java/dev/langchain4j/model/zhipu/ZhipuAiChatModel.java index 7c887e245..3c1436d33 100644 --- a/common/src/main/java/dev/langchain4j/model/zhipu/ZhipuAiChatModel.java +++ b/common/src/main/java/dev/langchain4j/model/zhipu/ZhipuAiChatModel.java @@ -26,8 +26,9 @@ import static dev.langchain4j.spi.ServiceHelper.loadFactories; import static java.util.Collections.singletonList; /** - * Represents an ZhipuAi language model with a chat completion interface, such as glm-3-turbo and glm-4. - * You can find description of parameters here. + * Represents an ZhipuAi language model with a chat completion interface, such as glm-3-turbo and + * glm-4. You can find description of parameters here. */ public class ZhipuAiChatModel implements ChatLanguageModel { @@ -49,24 +50,25 @@ public class ZhipuAiChatModel implements ChatLanguageModel { Integer maxRetries, Integer maxToken, Boolean logRequests, - Boolean logResponses - ) { + Boolean logResponses) { this.baseUrl = getOrDefault(baseUrl, "https://open.bigmodel.cn/"); this.temperature = getOrDefault(temperature, 0.7); this.topP = topP; this.model = getOrDefault(model, ChatCompletionModel.GLM_4.toString()); this.maxRetries = getOrDefault(maxRetries, 3); this.maxToken = getOrDefault(maxToken, 512); - this.client = ZhipuAiClient.builder() - .baseUrl(this.baseUrl) - .apiKey(apiKey) - .logRequests(getOrDefault(logRequests, false)) - .logResponses(getOrDefault(logResponses, false)) - .build(); + this.client = + ZhipuAiClient.builder() + .baseUrl(this.baseUrl) + .apiKey(apiKey) + .logRequests(getOrDefault(logRequests, false)) + .logResponses(getOrDefault(logResponses, false)) + .build(); } public static ZhipuAiChatModelBuilder builder() { - for (ZhipuAiChatModelBuilderFactory factories : loadFactories(ZhipuAiChatModelBuilderFactory.class)) { + for (ZhipuAiChatModelBuilderFactory factories : + loadFactories(ZhipuAiChatModelBuilderFactory.class)) { return factories.get(); } return new ZhipuAiChatModelBuilder(); @@ -78,36 +80,36 @@ public class ZhipuAiChatModel implements ChatLanguageModel { } @Override - public Response generate(List messages, List toolSpecifications) { + public Response generate( + List messages, List toolSpecifications) { ensureNotEmpty(messages, "messages"); - ChatCompletionRequest.Builder requestBuilder = ChatCompletionRequest.builder() - .model(this.model) - .maxTokens(maxToken) - .stream(false) - .topP(topP) - .toolChoice(AUTO) - .messages(toZhipuAiMessages(messages)); + ChatCompletionRequest.Builder requestBuilder = + ChatCompletionRequest.builder().model(this.model).maxTokens(maxToken).stream(false) + .topP(topP) + .toolChoice(AUTO) + .messages(toZhipuAiMessages(messages)); if (!isNullOrEmpty(toolSpecifications)) { requestBuilder.tools(toTools(toolSpecifications)); } - ChatCompletionResponse response = withRetry(() -> client.chatCompletion(requestBuilder.build()), maxRetries); + ChatCompletionResponse response = + withRetry(() -> client.chatCompletion(requestBuilder.build()), maxRetries); return Response.from( aiMessageFrom(response), tokenUsageFrom(response.getUsage()), - finishReasonFrom(response.getChoices().get(0).getFinishReason()) - ); + finishReasonFrom(response.getChoices().get(0).getFinishReason())); } @Override - public Response generate(List messages, ToolSpecification toolSpecification) { - return generate(messages, toolSpecification != null ? singletonList(toolSpecification) : null); + public Response generate( + List messages, ToolSpecification toolSpecification) { + return generate( + messages, toolSpecification != null ? singletonList(toolSpecification) : null); } public static class ZhipuAiChatModelBuilder { - public ZhipuAiChatModelBuilder() { - } + public ZhipuAiChatModelBuilder() {} } } diff --git a/common/src/main/java/dev/langchain4j/provider/AzureModelFactory.java b/common/src/main/java/dev/langchain4j/provider/AzureModelFactory.java index 23b76a929..5defb178b 100644 --- a/common/src/main/java/dev/langchain4j/provider/AzureModelFactory.java +++ b/common/src/main/java/dev/langchain4j/provider/AzureModelFactory.java @@ -20,27 +20,36 @@ public class AzureModelFactory implements ModelFactory, InitializingBean { @Override public ChatLanguageModel createChatModel(ChatModelConfig modelConfig) { - AzureOpenAiChatModel.Builder builder = AzureOpenAiChatModel.builder() - .endpoint(modelConfig.getBaseUrl()) - .apiKey(modelConfig.getApiKey()) - .deploymentName(modelConfig.getModelName()) - .temperature(modelConfig.getTemperature()) - .maxRetries(modelConfig.getMaxRetries()) - .topP(modelConfig.getTopP()) - .timeout(Duration.ofSeconds(modelConfig.getTimeOut() == null ? 0L : modelConfig.getTimeOut())) - .logRequestsAndResponses(modelConfig.getLogRequests() != null && modelConfig.getLogResponses()); + AzureOpenAiChatModel.Builder builder = + AzureOpenAiChatModel.builder() + .endpoint(modelConfig.getBaseUrl()) + .apiKey(modelConfig.getApiKey()) + .deploymentName(modelConfig.getModelName()) + .temperature(modelConfig.getTemperature()) + .maxRetries(modelConfig.getMaxRetries()) + .topP(modelConfig.getTopP()) + .timeout( + Duration.ofSeconds( + modelConfig.getTimeOut() == null + ? 0L + : modelConfig.getTimeOut())) + .logRequestsAndResponses( + modelConfig.getLogRequests() != null + && modelConfig.getLogResponses()); return builder.build(); } @Override public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModelConfig) { - AzureOpenAiEmbeddingModel.Builder builder = AzureOpenAiEmbeddingModel.builder() - .endpoint(embeddingModelConfig.getBaseUrl()) - .apiKey(embeddingModelConfig.getApiKey()) - .deploymentName(embeddingModelConfig.getModelName()) - .maxRetries(embeddingModelConfig.getMaxRetries()) - .logRequestsAndResponses(embeddingModelConfig.getLogRequests() != null - && embeddingModelConfig.getLogResponses()); + AzureOpenAiEmbeddingModel.Builder builder = + AzureOpenAiEmbeddingModel.builder() + .endpoint(embeddingModelConfig.getBaseUrl()) + .apiKey(embeddingModelConfig.getApiKey()) + .deploymentName(embeddingModelConfig.getModelName()) + .maxRetries(embeddingModelConfig.getMaxRetries()) + .logRequestsAndResponses( + embeddingModelConfig.getLogRequests() != null + && embeddingModelConfig.getLogResponses()); return builder.build(); } diff --git a/common/src/main/java/dev/langchain4j/provider/DashscopeModelFactory.java b/common/src/main/java/dev/langchain4j/provider/DashscopeModelFactory.java index d946d80b5..e2447af14 100644 --- a/common/src/main/java/dev/langchain4j/provider/DashscopeModelFactory.java +++ b/common/src/main/java/dev/langchain4j/provider/DashscopeModelFactory.java @@ -23,8 +23,10 @@ public class DashscopeModelFactory implements ModelFactory, InitializingBean { .baseUrl(modelConfig.getBaseUrl()) .apiKey(modelConfig.getApiKey()) .modelName(modelConfig.getModelName()) - .temperature(modelConfig.getTemperature() == null ? 0L : - modelConfig.getTemperature().floatValue()) + .temperature( + modelConfig.getTemperature() == null + ? 0L + : modelConfig.getTemperature().floatValue()) .topP(modelConfig.getTopP()) .enableSearch(modelConfig.getEnableSearch()) .build(); diff --git a/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java b/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java index b9b3eda94..ccadba509 100644 --- a/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java +++ b/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java @@ -11,6 +11,6 @@ public class EmbeddingModelConstant { public static final String BGE_SMALL_ZH = "bge-small-zh"; public static final String ALL_MINILM_L6_V2 = "all-minilm-l6-v2-q"; public static final EmbeddingModel BGE_SMALL_ZH_MODEL = new BgeSmallZhEmbeddingModel(); - public static final EmbeddingModel ALL_MINI_LM_L6_V2_MODEL = new AllMiniLmL6V2QuantizedEmbeddingModel(); - + public static final EmbeddingModel ALL_MINI_LM_L6_V2_MODEL = + new AllMiniLmL6V2QuantizedEmbeddingModel(); } diff --git a/common/src/main/java/dev/langchain4j/provider/LocalAiModelFactory.java b/common/src/main/java/dev/langchain4j/provider/LocalAiModelFactory.java index 25cda09f3..81c40b9b2 100644 --- a/common/src/main/java/dev/langchain4j/provider/LocalAiModelFactory.java +++ b/common/src/main/java/dev/langchain4j/provider/LocalAiModelFactory.java @@ -16,10 +16,10 @@ public class LocalAiModelFactory implements ModelFactory, InitializingBean { public static final String PROVIDER = "LOCAL_AI"; public static final String DEFAULT_BASE_URL = "http://localhost:8080"; public static final String DEFAULT_MODEL_NAME = "ggml-gpt4all-j"; + @Override public ChatLanguageModel createChatModel(ChatModelConfig modelConfig) { - return LocalAiChatModel - .builder() + return LocalAiChatModel.builder() .baseUrl(modelConfig.getBaseUrl()) .modelName(modelConfig.getModelName()) .temperature(modelConfig.getTemperature()) @@ -46,4 +46,4 @@ public class LocalAiModelFactory implements ModelFactory, InitializingBean { public void afterPropertiesSet() { ModelProvider.add(PROVIDER, this); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/provider/ModelProvider.java b/common/src/main/java/dev/langchain4j/provider/ModelProvider.java index 8589bb462..fdce1497b 100644 --- a/common/src/main/java/dev/langchain4j/provider/ModelProvider.java +++ b/common/src/main/java/dev/langchain4j/provider/ModelProvider.java @@ -25,10 +25,11 @@ public class ModelProvider { } public static ChatLanguageModel getChatModel(ChatModelConfig modelConfig) { - if (modelConfig == null || StringUtils.isBlank(modelConfig.getProvider()) + if (modelConfig == null + || StringUtils.isBlank(modelConfig.getProvider()) || StringUtils.isBlank(modelConfig.getBaseUrl())) { - ChatModelParameterConfig parameterConfig = ContextUtils.getBean( - ChatModelParameterConfig.class); + ChatModelParameterConfig parameterConfig = + ContextUtils.getBean(ChatModelParameterConfig.class); modelConfig = parameterConfig.convert(); } ModelFactory modelFactory = factories.get(modelConfig.getProvider().toUpperCase()); @@ -36,7 +37,8 @@ public class ModelProvider { return modelFactory.createChatModel(modelConfig); } - throw new RuntimeException("Unsupported ChatLanguageModel provider: " + modelConfig.getProvider()); + throw new RuntimeException( + "Unsupported ChatLanguageModel provider: " + modelConfig.getProvider()); } public static EmbeddingModel getEmbeddingModel() { @@ -45,8 +47,8 @@ public class ModelProvider { public static EmbeddingModel getEmbeddingModel(EmbeddingModelConfig embeddingModel) { if (embeddingModel == null || StringUtils.isBlank(embeddingModel.getProvider())) { - EmbeddingModelParameterConfig parameterConfig = ContextUtils.getBean( - EmbeddingModelParameterConfig.class); + EmbeddingModelParameterConfig parameterConfig = + ContextUtils.getBean(EmbeddingModelParameterConfig.class); embeddingModel = parameterConfig.convert(); } ModelFactory modelFactory = factories.get(embeddingModel.getProvider().toUpperCase()); @@ -54,6 +56,7 @@ public class ModelProvider { return modelFactory.createEmbeddingModel(embeddingModel); } - throw new RuntimeException("Unsupported EmbeddingModel provider: " + embeddingModel.getProvider()); + throw new RuntimeException( + "Unsupported EmbeddingModel provider: " + embeddingModel.getProvider()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/provider/OllamaModelFactory.java b/common/src/main/java/dev/langchain4j/provider/OllamaModelFactory.java index e69cded99..c34b4882e 100644 --- a/common/src/main/java/dev/langchain4j/provider/OllamaModelFactory.java +++ b/common/src/main/java/dev/langchain4j/provider/OllamaModelFactory.java @@ -21,8 +21,7 @@ public class OllamaModelFactory implements ModelFactory, InitializingBean { @Override public ChatLanguageModel createChatModel(ChatModelConfig modelConfig) { - return OllamaChatModel - .builder() + return OllamaChatModel.builder() .baseUrl(modelConfig.getBaseUrl()) .modelName(modelConfig.getModelName()) .temperature(modelConfig.getTemperature()) diff --git a/common/src/main/java/dev/langchain4j/provider/OpenAiModelFactory.java b/common/src/main/java/dev/langchain4j/provider/OpenAiModelFactory.java index d4c4ec38d..5fcd45261 100644 --- a/common/src/main/java/dev/langchain4j/provider/OpenAiModelFactory.java +++ b/common/src/main/java/dev/langchain4j/provider/OpenAiModelFactory.java @@ -21,8 +21,7 @@ public class OpenAiModelFactory implements ModelFactory, InitializingBean { @Override public ChatLanguageModel createChatModel(ChatModelConfig modelConfig) { - return OpenAiChatModel - .builder() + return OpenAiChatModel.builder() .baseUrl(modelConfig.getBaseUrl()) .modelName(modelConfig.getModelName()) .apiKey(modelConfig.keyDecrypt()) diff --git a/common/src/main/java/dev/langchain4j/provider/ZhipuModelFactory.java b/common/src/main/java/dev/langchain4j/provider/ZhipuModelFactory.java index ab99b8149..0f004d424 100644 --- a/common/src/main/java/dev/langchain4j/provider/ZhipuModelFactory.java +++ b/common/src/main/java/dev/langchain4j/provider/ZhipuModelFactory.java @@ -16,6 +16,7 @@ public class ZhipuModelFactory implements ModelFactory, InitializingBean { public static final String DEFAULT_BASE_URL = "https://open.bigmodel.cn/"; public static final String DEFAULT_MODEL_NAME = ChatCompletionModel.GLM_4.toString(); public static final String DEFAULT_EMBEDDING_MODEL_NAME = "embedding-2"; + @Override public ChatLanguageModel createChatModel(ChatModelConfig modelConfig) { return ZhipuAiChatModel.builder() diff --git a/common/src/main/java/dev/langchain4j/qianfan/spring/ChatModelProperties.java b/common/src/main/java/dev/langchain4j/qianfan/spring/ChatModelProperties.java index 55870752c..9ec121214 100644 --- a/common/src/main/java/dev/langchain4j/qianfan/spring/ChatModelProperties.java +++ b/common/src/main/java/dev/langchain4j/qianfan/spring/ChatModelProperties.java @@ -18,4 +18,4 @@ class ChatModelProperties { private Double penaltyScore; private Boolean logRequests; private Boolean logResponses; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/qianfan/spring/EmbeddingModelProperties.java b/common/src/main/java/dev/langchain4j/qianfan/spring/EmbeddingModelProperties.java index 89521d3ca..b3d3defff 100644 --- a/common/src/main/java/dev/langchain4j/qianfan/spring/EmbeddingModelProperties.java +++ b/common/src/main/java/dev/langchain4j/qianfan/spring/EmbeddingModelProperties.java @@ -15,5 +15,4 @@ class EmbeddingModelProperties { private String user; private Boolean logRequests; private Boolean logResponses; - -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/qianfan/spring/LanguageModelProperties.java b/common/src/main/java/dev/langchain4j/qianfan/spring/LanguageModelProperties.java index e2dad2b4c..39c57096e 100644 --- a/common/src/main/java/dev/langchain4j/qianfan/spring/LanguageModelProperties.java +++ b/common/src/main/java/dev/langchain4j/qianfan/spring/LanguageModelProperties.java @@ -18,5 +18,4 @@ class LanguageModelProperties { private Double penaltyScore; private Boolean logRequests; private Boolean logResponses; - -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/qianfan/spring/Properties.java b/common/src/main/java/dev/langchain4j/qianfan/spring/Properties.java index 0ab2c5c5d..dfdb8b41c 100644 --- a/common/src/main/java/dev/langchain4j/qianfan/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/qianfan/spring/Properties.java @@ -12,18 +12,13 @@ public class Properties { static final String PREFIX = "langchain4j.qianfan"; - @NestedConfigurationProperty - ChatModelProperties chatModel; + @NestedConfigurationProperty ChatModelProperties chatModel; - @NestedConfigurationProperty - ChatModelProperties streamingChatModel; + @NestedConfigurationProperty ChatModelProperties streamingChatModel; - @NestedConfigurationProperty - LanguageModelProperties languageModel; + @NestedConfigurationProperty LanguageModelProperties languageModel; - @NestedConfigurationProperty - LanguageModelProperties streamingLanguageModel; + @NestedConfigurationProperty LanguageModelProperties streamingLanguageModel; - @NestedConfigurationProperty - EmbeddingModelProperties embeddingModel; -} \ No newline at end of file + @NestedConfigurationProperty EmbeddingModelProperties embeddingModel; +} diff --git a/common/src/main/java/dev/langchain4j/qianfan/spring/QianfanAutoConfig.java b/common/src/main/java/dev/langchain4j/qianfan/spring/QianfanAutoConfig.java index 1d7ecd6d5..7bef69e52 100644 --- a/common/src/main/java/dev/langchain4j/qianfan/spring/QianfanAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/qianfan/spring/QianfanAutoConfig.java @@ -1,7 +1,5 @@ package dev.langchain4j.qianfan.spring; -import static dev.langchain4j.qianfan.spring.Properties.PREFIX; - import dev.langchain4j.model.qianfan.QianfanChatModel; import dev.langchain4j.model.qianfan.QianfanEmbeddingModel; import dev.langchain4j.model.qianfan.QianfanLanguageModel; @@ -12,6 +10,8 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static dev.langchain4j.qianfan.spring.Properties.PREFIX; + @Configuration @EnableConfigurationProperties(Properties.class) public class QianfanAutoConfig { @@ -111,4 +111,4 @@ public class QianfanAutoConfig { .logResponses(embeddingModelProperties.getLogResponses()) .build(); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/store/embedding/BaseEmbeddingStoreFactory.java b/common/src/main/java/dev/langchain4j/store/embedding/BaseEmbeddingStoreFactory.java index 2c92f1a24..803374943 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/BaseEmbeddingStoreFactory.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/BaseEmbeddingStoreFactory.java @@ -6,11 +6,12 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; public abstract class BaseEmbeddingStoreFactory implements EmbeddingStoreFactory { - protected final Map> collectionNameToStore = new ConcurrentHashMap<>(); + protected final Map> collectionNameToStore = + new ConcurrentHashMap<>(); public EmbeddingStore create(String collectionName) { return collectionNameToStore.computeIfAbsent(collectionName, this::createEmbeddingStore); } public abstract EmbeddingStore createEmbeddingStore(String collectionName); -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingCollection.java b/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingCollection.java index 03f11fa4a..87a2d5a9c 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingCollection.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingCollection.java @@ -1,6 +1,5 @@ package dev.langchain4j.store.embedding; - import lombok.Data; import java.util.Map; @@ -13,5 +12,4 @@ public class EmbeddingCollection { private String name; private Map metaData; - } diff --git a/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactory.java b/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactory.java index c808bbe6c..0150625c7 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactory.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactory.java @@ -5,5 +5,4 @@ import dev.langchain4j.data.segment.TextSegment; public interface EmbeddingStoreFactory { EmbeddingStore create(String collectionName); - -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactoryProvider.java b/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactoryProvider.java index dcac744e0..234449d0b 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactoryProvider.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/EmbeddingStoreFactoryProvider.java @@ -12,29 +12,39 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; public class EmbeddingStoreFactoryProvider { - protected static final Map factoryMap = new ConcurrentHashMap<>(); + protected static final Map factoryMap = + new ConcurrentHashMap<>(); public static EmbeddingStoreFactory getFactory() { - EmbeddingStoreParameterConfig parameterConfig = ContextUtils.getBean(EmbeddingStoreParameterConfig.class); + EmbeddingStoreParameterConfig parameterConfig = + ContextUtils.getBean(EmbeddingStoreParameterConfig.class); return getFactory(parameterConfig.convert()); } public static EmbeddingStoreFactory getFactory(EmbeddingStoreConfig embeddingStoreConfig) { - if (embeddingStoreConfig == null || StringUtils.isBlank(embeddingStoreConfig.getProvider())) { + if (embeddingStoreConfig == null + || StringUtils.isBlank(embeddingStoreConfig.getProvider())) { return ContextUtils.getBean(EmbeddingStoreFactory.class); } if (EmbeddingStoreType.CHROMA.name().equalsIgnoreCase(embeddingStoreConfig.getProvider())) { - return factoryMap.computeIfAbsent(embeddingStoreConfig, + return factoryMap.computeIfAbsent( + embeddingStoreConfig, storeConfig -> new ChromaEmbeddingStoreFactory(storeConfig)); } if (EmbeddingStoreType.MILVUS.name().equalsIgnoreCase(embeddingStoreConfig.getProvider())) { - return factoryMap.computeIfAbsent(embeddingStoreConfig, + return factoryMap.computeIfAbsent( + embeddingStoreConfig, storeConfig -> new MilvusEmbeddingStoreFactory(storeConfig)); } - if (EmbeddingStoreType.IN_MEMORY.name().equalsIgnoreCase(embeddingStoreConfig.getProvider())) { - return factoryMap.computeIfAbsent(embeddingStoreConfig, + if (EmbeddingStoreType.IN_MEMORY + .name() + .equalsIgnoreCase(embeddingStoreConfig.getProvider())) { + return factoryMap.computeIfAbsent( + embeddingStoreConfig, storeConfig -> new InMemoryEmbeddingStoreFactory(storeConfig)); } - throw new RuntimeException("Unsupported EmbeddingStoreFactory provider: " + embeddingStoreConfig.getProvider()); + throw new RuntimeException( + "Unsupported EmbeddingStoreFactory provider: " + + embeddingStoreConfig.getProvider()); } -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/store/embedding/Retrieval.java b/common/src/main/java/dev/langchain4j/store/embedding/Retrieval.java index e126a4ad0..6a04df504 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/Retrieval.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/Retrieval.java @@ -35,8 +35,9 @@ public class Retrieval { return false; } Retrieval retrieval = (Retrieval) o; - return Double.compare(retrieval.distance, distance) == 0 && Objects.equal(id, - retrieval.id) && Objects.equal(query, retrieval.query) + return Double.compare(retrieval.distance, distance) == 0 + && Objects.equal(id, retrieval.id) + && Objects.equal(query, retrieval.query) && Objects.equal(metadata, retrieval.metadata); } diff --git a/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQuery.java b/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQuery.java index 6ba73cfc5..d634f461c 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQuery.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQuery.java @@ -15,6 +15,4 @@ public class RetrieveQuery { private Map filterCondition; private List> queryEmbeddings; - - } diff --git a/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQueryResult.java b/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQueryResult.java index 194888714..49e125290 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQueryResult.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/RetrieveQueryResult.java @@ -1,6 +1,5 @@ package dev.langchain4j.store.embedding; - import lombok.Data; import java.util.List; @@ -11,5 +10,4 @@ public class RetrieveQueryResult { private String query; private List retrieval; - } diff --git a/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java b/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java index 2deb3de40..938f99fe9 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java @@ -1,6 +1,5 @@ package dev.langchain4j.store.embedding; - import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.common.pojo.DataItem; import dev.langchain4j.data.document.Metadata; @@ -18,12 +17,20 @@ public class TextSegmentConvert { public static final String QUERY_ID = "queryId"; public static List convertToEmbedding(List dataItems) { - return dataItems.stream().map(dataItem -> { - Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class); - TextSegment textSegment = TextSegment.from(dataItem.getName(), new Metadata(meta)); - addQueryId(textSegment, dataItem.getId() + dataItem.getType().name().toLowerCase()); - return textSegment; - }).collect(Collectors.toList()); + return dataItems.stream() + .map( + dataItem -> { + Map meta = + JSONObject.parseObject( + JSONObject.toJSONString(dataItem), Map.class); + TextSegment textSegment = + TextSegment.from(dataItem.getName(), new Metadata(meta)); + addQueryId( + textSegment, + dataItem.getId() + dataItem.getType().name().toLowerCase()); + return textSegment; + }) + .collect(Collectors.toList()); } public static void addQueryId(TextSegment textSegment, String queryId) { diff --git a/common/src/main/java/dev/langchain4j/store/embedding/inmemory/InMemoryEmbeddingStore.java b/common/src/main/java/dev/langchain4j/store/embedding/inmemory/InMemoryEmbeddingStore.java index 05c1475d7..237042c9e 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/inmemory/InMemoryEmbeddingStore.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/inmemory/InMemoryEmbeddingStore.java @@ -39,15 +39,17 @@ import static java.util.stream.Collectors.toList; /** * An {@link EmbeddingStore} that stores embeddings in memory. - *

- * Uses a brute force approach by iterating over all embeddings to find the best matches. - *

- * This store can be persisted using the {@link #serializeToJson()} and {@link #serializeToFile(Path)} methods. - *

- * It can also be recreated from JSON or a file using the {@link #fromJson(String)} and {@link #fromFile(Path)} methods. * - * @param The class of the object that has been embedded. - * Typically, it is {@link dev.langchain4j.data.segment.TextSegment}. + *

Uses a brute force approach by iterating over all embeddings to find the best matches. + * + *

This store can be persisted using the {@link #serializeToJson()} and {@link + * #serializeToFile(Path)} methods. + * + *

It can also be recreated from JSON or a file using the {@link #fromJson(String)} and {@link + * #fromFile(Path)} methods. + * + * @param The class of the object that has been embedded. Typically, it is {@link + * dev.langchain4j.data.segment.TextSegment}. */ public class InMemoryEmbeddingStore implements EmbeddingStore { @@ -80,17 +82,16 @@ public class InMemoryEmbeddingStore implements EmbeddingStore entry.id) - .collect(toList()); + return newEntries.stream().map(entry -> entry.id).collect(toList()); } @Override public List addAll(List embeddings) { - List> newEntries = embeddings.stream() - .map(embedding -> new Entry(randomUUID(), embedding)) - .collect(toList()); + List> newEntries = + embeddings.stream() + .map(embedding -> new Entry(randomUUID(), embedding)) + .collect(toList()); return add(newEntries); } @@ -98,12 +99,15 @@ public class InMemoryEmbeddingStore implements EmbeddingStore addAll(List embeddings, List embedded) { if (embeddings.size() != embedded.size()) { - throw new IllegalArgumentException("The list of embeddings and embedded must have the same size"); + throw new IllegalArgumentException( + "The list of embeddings and embedded must have the same size"); } - List> newEntries = IntStream.range(0, embeddings.size()) - .mapToObj(i -> new Entry<>(randomUUID(), embeddings.get(i), embedded.get(i))) - .collect(toList()); + List> newEntries = + IntStream.range(0, embeddings.size()) + .mapToObj( + i -> new Entry<>(randomUUID(), embeddings.get(i), embedded.get(i))) + .collect(toList()); return add(newEntries); } @@ -119,15 +123,16 @@ public class InMemoryEmbeddingStore implements EmbeddingStore { - if (entry.embedded instanceof TextSegment) { - return filter.test(((TextSegment) entry.embedded).metadata()); - } else if (entry.embedded == null) { - return false; - } else { - throw new UnsupportedOperationException("Not supported yet."); - } - }); + entries.removeIf( + entry -> { + if (entry.embedded instanceof TextSegment) { + return filter.test(((TextSegment) entry.embedded).metadata()); + } else if (entry.embedded == null) { + return false; + } else { + throw new UnsupportedOperationException("Not supported yet."); + } + }); } @Override @@ -152,8 +157,9 @@ public class InMemoryEmbeddingStore implements EmbeddingStore= embeddingSearchRequest.minScore()) { matches.add(new EmbeddingMatch<>(score, entry.id, entry.embedding, entry.embedded)); diff --git a/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java b/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java index 04b6bbef5..085577a2a 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java @@ -42,12 +42,10 @@ import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; /** - * Represents an Milvus index as an embedding store. - *
- * Supports both local and managed Milvus instances. - *
- * Supports storing {@link Metadata} and filtering by it using a {@link Filter} - * (provided inside an {@link EmbeddingSearchRequest}). + * Represents an Milvus index as an embedding store.
+ * Supports both local and managed Milvus instances.
+ * Supports storing {@link Metadata} and filtering by it using a {@link Filter} (provided inside an + * {@link EmbeddingSearchRequest}). */ public class MilvusEmbeddingStore implements EmbeddingStore { @@ -78,15 +76,14 @@ public class MilvusEmbeddingStore implements EmbeddingStore { ConsistencyLevelEnum consistencyLevel, Boolean retrieveEmbeddingsOnSearch, Boolean autoFlushOnInsert, - String databaseName - ) { - ConnectParam.Builder connectBuilder = ConnectParam - .newBuilder() - .withHost(getOrDefault(host, "localhost")) - .withPort(getOrDefault(port, 19530)) - .withUri(uri) - .withToken(token) - .withAuthorization(username, password); + String databaseName) { + ConnectParam.Builder connectBuilder = + ConnectParam.newBuilder() + .withHost(getOrDefault(host, "localhost")) + .withPort(getOrDefault(port, 19530)) + .withUri(uri) + .withToken(token) + .withAuthorization(username, password); if (databaseName != null) { connectBuilder.withDatabaseName(databaseName); @@ -99,8 +96,13 @@ public class MilvusEmbeddingStore implements EmbeddingStore { this.retrieveEmbeddingsOnSearch = getOrDefault(retrieveEmbeddingsOnSearch, false); this.autoFlushOnInsert = getOrDefault(autoFlushOnInsert, false); if (!hasCollection(milvusClient, this.collectionName)) { - createCollection(milvusClient, this.collectionName, ensureNotNull(dimension, "dimension")); - createIndex(milvusClient, this.collectionName, getOrDefault(indexType, FLAT), this.metricType); + createCollection( + milvusClient, this.collectionName, ensureNotNull(dimension, "dimension")); + createIndex( + milvusClient, + this.collectionName, + getOrDefault(indexType, FLAT), + this.metricType); } loadCollectionInMemory(milvusClient, collectionName); @@ -139,30 +141,33 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } @Override - public EmbeddingSearchResult search(EmbeddingSearchRequest embeddingSearchRequest) { + public EmbeddingSearchResult search( + EmbeddingSearchRequest embeddingSearchRequest) { - SearchParam searchParam = buildSearchRequest( - collectionName, - embeddingSearchRequest.queryEmbedding().vectorAsList(), - embeddingSearchRequest.filter(), - embeddingSearchRequest.maxResults(), - metricType, - consistencyLevel - ); + SearchParam searchParam = + buildSearchRequest( + collectionName, + embeddingSearchRequest.queryEmbedding().vectorAsList(), + embeddingSearchRequest.filter(), + embeddingSearchRequest.maxResults(), + metricType, + consistencyLevel); - SearchResultsWrapper resultsWrapper = CollectionOperationsExecutor.search(milvusClient, searchParam); + SearchResultsWrapper resultsWrapper = + CollectionOperationsExecutor.search(milvusClient, searchParam); - List> matches = toEmbeddingMatches( - milvusClient, - resultsWrapper, - collectionName, - consistencyLevel, - retrieveEmbeddingsOnSearch - ); + List> matches = + toEmbeddingMatches( + milvusClient, + resultsWrapper, + collectionName, + consistencyLevel, + retrieveEmbeddingsOnSearch); - List> result = matches.stream() - .filter(match -> match.score() >= embeddingSearchRequest.minScore()) - .collect(toList()); + List> result = + matches.stream() + .filter(match -> match.score() >= embeddingSearchRequest.minScore()) + .collect(toList()); return new EmbeddingSearchResult<>(result); } @@ -171,15 +176,17 @@ public class MilvusEmbeddingStore implements EmbeddingStore { addAllInternal( singletonList(id), singletonList(embedding), - textSegment == null ? null : singletonList(textSegment) - ); + textSegment == null ? null : singletonList(textSegment)); } - private void addAllInternal(List ids, List embeddings, List textSegments) { + private void addAllInternal( + List ids, List embeddings, List textSegments) { List fields = new ArrayList<>(); fields.add(new InsertParam.Field(ID_FIELD_NAME, ids)); fields.add(new InsertParam.Field(TEXT_FIELD_NAME, toScalars(textSegments, ids.size()))); - fields.add(new InsertParam.Field(METADATA_FIELD_NAME, toMetadataJsons(textSegments, ids.size()))); + fields.add( + new InsertParam.Field( + METADATA_FIELD_NAME, toMetadataJsons(textSegments, ids.size()))); fields.add(new InsertParam.Field(VECTOR_FIELD_NAME, toVectors(embeddings))); insert(milvusClient, collectionName, fields); @@ -210,8 +217,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { private String databaseName; /** - * @param host The host of the self-managed Milvus instance. - * Default value: "localhost". + * @param host The host of the self-managed Milvus instance. Default value: "localhost". * @return builder */ public Builder host(String host) { @@ -220,8 +226,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param port The port of the self-managed Milvus instance. - * Default value: 19530. + * @param port The port of the self-managed Milvus instance. Default value: 19530. * @return builder */ public Builder port(Integer port) { @@ -230,9 +235,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param collectionName The name of the Milvus collection. - * If there is no such collection yet, it will be created automatically. - * Default value: "default". + * @param collectionName The name of the Milvus collection. If there is no such collection + * yet, it will be created automatically. Default value: "default". * @return builder */ public Builder collectionName(String collectionName) { @@ -241,8 +245,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param dimension The dimension of the embedding vector. (e.g. 384) - * Mandatory if a new collection should be created. + * @param dimension The dimension of the embedding vector. (e.g. 384) Mandatory if a new + * collection should be created. * @return builder */ public Builder dimension(Integer dimension) { @@ -251,8 +255,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param indexType The type of the index. - * Default value: FLAT. + * @param indexType The type of the index. Default value: FLAT. * @return builder */ public Builder indexType(IndexType indexType) { @@ -261,8 +264,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param metricType The type of the metric used for similarity search. - * Default value: COSINE. + * @param metricType The type of the metric used for similarity search. Default value: + * COSINE. * @return builder */ public Builder metricType(MetricType metricType) { @@ -271,7 +274,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param uri The URI of the managed Milvus instance. (e.g. "https://xxx.api.gcp-us-west1.zillizcloud.com") + * @param uri The URI of the managed Milvus instance. (e.g. + * "https://xxx.api.gcp-us-west1.zillizcloud.com") * @return builder */ public Builder uri(String uri) { @@ -289,7 +293,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param username The username. See details here. + * @param username The username. See details here. * @return builder */ public Builder username(String username) { @@ -298,7 +303,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param password The password. See details here. + * @param password The password. See details here. * @return builder */ public Builder password(String password) { @@ -307,8 +313,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param consistencyLevel The consistency level used by Milvus. - * Default value: EVENTUALLY. + * @param consistencyLevel The consistency level used by Milvus. Default value: EVENTUALLY. * @return builder */ public Builder consistencyLevel(ConsistencyLevelEnum consistencyLevel) { @@ -317,12 +322,11 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param retrieveEmbeddingsOnSearch During a similarity search in Milvus (when calling findRelevant()), - * the embedding itself is not retrieved. - * To retrieve the embedding, an additional query is required. - * Setting this parameter to "true" will ensure that embedding is retrieved. - * Be aware that this will impact the performance of the search. - * Default value: false. + * @param retrieveEmbeddingsOnSearch During a similarity search in Milvus (when calling + * findRelevant()), the embedding itself is not retrieved. To retrieve the embedding, an + * additional query is required. Setting this parameter to "true" will ensure that + * embedding is retrieved. Be aware that this will impact the performance of the search. + * Default value: false. * @return builder */ public Builder retrieveEmbeddingsOnSearch(Boolean retrieveEmbeddingsOnSearch) { @@ -331,11 +335,10 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param autoFlushOnInsert Whether to automatically flush after each insert - * ({@code add(...)} or {@code addAll(...)} methods). - * Default value: false. - * More info can be found - * here. + * @param autoFlushOnInsert Whether to automatically flush after each insert ({@code + * add(...)} or {@code addAll(...)} methods). Default value: false. More info can be + * found here. * @return builder */ public Builder autoFlushOnInsert(Boolean autoFlushOnInsert) { @@ -344,8 +347,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore { } /** - * @param databaseName Milvus name of database. - * Default value: null. In this case default Milvus database name will be used. + * @param databaseName Milvus name of database. Default value: null. In this case default + * Milvus database name will be used. * @return builder */ public Builder databaseName(String databaseName) { @@ -368,8 +371,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { consistencyLevel, retrieveEmbeddingsOnSearch, autoFlushOnInsert, - databaseName - ); + databaseName); } } } diff --git a/common/src/main/java/dev/langchain4j/zhipu/spring/ChatModelProperties.java b/common/src/main/java/dev/langchain4j/zhipu/spring/ChatModelProperties.java index ceaca061d..c2b0e05c6 100644 --- a/common/src/main/java/dev/langchain4j/zhipu/spring/ChatModelProperties.java +++ b/common/src/main/java/dev/langchain4j/zhipu/spring/ChatModelProperties.java @@ -16,4 +16,4 @@ class ChatModelProperties { Integer maxToken; Boolean logRequests; Boolean logResponses; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/zhipu/spring/EmbeddingModelProperties.java b/common/src/main/java/dev/langchain4j/zhipu/spring/EmbeddingModelProperties.java index 8b79cccfb..4e67756e6 100644 --- a/common/src/main/java/dev/langchain4j/zhipu/spring/EmbeddingModelProperties.java +++ b/common/src/main/java/dev/langchain4j/zhipu/spring/EmbeddingModelProperties.java @@ -13,4 +13,4 @@ class EmbeddingModelProperties { Integer maxRetries; Boolean logRequests; Boolean logResponses; -} \ No newline at end of file +} diff --git a/common/src/main/java/dev/langchain4j/zhipu/spring/Properties.java b/common/src/main/java/dev/langchain4j/zhipu/spring/Properties.java index 97eeb55d9..68bd040c2 100644 --- a/common/src/main/java/dev/langchain4j/zhipu/spring/Properties.java +++ b/common/src/main/java/dev/langchain4j/zhipu/spring/Properties.java @@ -12,12 +12,9 @@ public class Properties { static final String PREFIX = "langchain4j.zhipu"; - @NestedConfigurationProperty - ChatModelProperties chatModel; + @NestedConfigurationProperty ChatModelProperties chatModel; - @NestedConfigurationProperty - ChatModelProperties streamingChatModel; + @NestedConfigurationProperty ChatModelProperties streamingChatModel; - @NestedConfigurationProperty - EmbeddingModelProperties embeddingModel; -} \ No newline at end of file + @NestedConfigurationProperty EmbeddingModelProperties embeddingModel; +} diff --git a/common/src/main/java/dev/langchain4j/zhipu/spring/ZhipuAutoConfig.java b/common/src/main/java/dev/langchain4j/zhipu/spring/ZhipuAutoConfig.java index d40f5b465..9b3ac945f 100644 --- a/common/src/main/java/dev/langchain4j/zhipu/spring/ZhipuAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/zhipu/spring/ZhipuAutoConfig.java @@ -1,7 +1,5 @@ package dev.langchain4j.zhipu.spring; -import static dev.langchain4j.zhipu.spring.Properties.PREFIX; - import dev.langchain4j.model.zhipu.ZhipuAiChatModel; import dev.langchain4j.model.zhipu.ZhipuAiEmbeddingModel; import dev.langchain4j.model.zhipu.ZhipuAiStreamingChatModel; @@ -10,6 +8,8 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static dev.langchain4j.zhipu.spring.Properties.PREFIX; + @Configuration @EnableConfigurationProperties(Properties.class) public class ZhipuAutoConfig { @@ -60,4 +60,4 @@ public class ZhipuAutoConfig { .logResponses(embeddingModelProperties.getLogResponses()) .build(); } -} \ No newline at end of file +} diff --git a/common/src/test/java/com/tencent/supersonic/common/DateUtilsTest.java b/common/src/test/java/com/tencent/supersonic/common/DateUtilsTest.java index 0dd80f6a9..c6e45d652 100644 --- a/common/src/test/java/com/tencent/supersonic/common/DateUtilsTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/DateUtilsTest.java @@ -48,8 +48,14 @@ class DateUtilsTest { String startDate = "2023-07-29"; String endDate = "2023-08-03"; List actualDateList = DateUtils.getDateList(startDate, endDate, Constants.DAY); - List expectedDateList = Lists.newArrayList("2023-07-29", "2023-07-30", - "2023-07-31", "2023-08-01", "2023-08-02", "2023-08-03"); + List expectedDateList = + Lists.newArrayList( + "2023-07-29", + "2023-07-30", + "2023-07-31", + "2023-08-01", + "2023-08-02", + "2023-08-03"); Assertions.assertEquals(actualDateList, expectedDateList); } @@ -58,8 +64,8 @@ class DateUtilsTest { String startDate = "2023-10-30"; String endDate = "2023-11-13"; List actualDateList = DateUtils.getDateList(startDate, endDate, Constants.WEEK); - List expectedDateList = Lists.newArrayList("2023-10-30", "2023-11-06", - "2023-11-13"); + List expectedDateList = + Lists.newArrayList("2023-10-30", "2023-11-06", "2023-11-13"); Assertions.assertEquals(actualDateList, expectedDateList); } @@ -68,7 +74,8 @@ class DateUtilsTest { String startDate = "2023-07-01"; String endDate = "2023-10-01"; List actualDateList = DateUtils.getDateList(startDate, endDate, Constants.MONTH); - List expectedDateList = Lists.newArrayList("2023-07", "2023-08", "2023-09", "2023-10"); + List expectedDateList = + Lists.newArrayList("2023-07", "2023-08", "2023-09", "2023-10"); Assertions.assertEquals(actualDateList, expectedDateList); } -} \ No newline at end of file +} diff --git a/common/src/test/java/com/tencent/supersonic/common/calcite/SqlParseUtilsTest.java b/common/src/test/java/com/tencent/supersonic/common/calcite/SqlParseUtilsTest.java index f52b39b25..106d8d6c9 100644 --- a/common/src/test/java/com/tencent/supersonic/common/calcite/SqlParseUtilsTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/calcite/SqlParseUtilsTest.java @@ -1,14 +1,14 @@ package com.tencent.supersonic.common.calcite; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.parser.SqlParseException; import org.junit.Assert; import org.junit.jupiter.api.Test; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + /** * SqlParseUtils Test * @@ -20,9 +20,10 @@ class SqlParseUtilsTest { @Test void addAliasToSql() throws SqlParseException { - String addAliasToSql = SqlParseUtils.addAliasToSql( - "select sum(pv) from ( select * from t_1 " - + "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1"); + String addAliasToSql = + SqlParseUtils.addAliasToSql( + "select sum(pv) from ( select * from t_1 " + + "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1"); Assert.assertTrue(addAliasToSql.toLowerCase().contains("as pv")); } @@ -30,85 +31,102 @@ class SqlParseUtilsTest { @Test void addFieldToSql() throws SqlParseException { - String addFieldToSql = SqlParseUtils.addFieldsToSql( - "select pv from ( select * from t_1 " - + "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1", - Collections.singletonList("uv")); + String addFieldToSql = + SqlParseUtils.addFieldsToSql( + "select pv from ( select * from t_1 " + + "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1", + Collections.singletonList("uv")); Assert.assertTrue(addFieldToSql.toLowerCase().contains("uv")); - addFieldToSql = SqlParseUtils.addFieldsToSql( - "select uv from ( select * from t_1 " - + "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 " - + "order by play_count desc limit 10", - Collections.singletonList("pv")); + addFieldToSql = + SqlParseUtils.addFieldsToSql( + "select uv from ( select * from t_1 " + + "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 " + + "order by play_count desc limit 10", + Collections.singletonList("pv")); Assert.assertTrue(addFieldToSql.toLowerCase().contains("pv")); - addFieldToSql = SqlParseUtils.addFieldsToSql( - "select uv from " - + "( select * from t_1 where sys_imp_date >= '2023-07-07' " - + " and sys_imp_date <= '2023-07-07' " - + ") as t_sub_1 " - + "where user_id = '张三' order by play_count desc limit 10", - Collections.singletonList("pv")); + addFieldToSql = + SqlParseUtils.addFieldsToSql( + "select uv from " + + "( select * from t_1 where sys_imp_date >= '2023-07-07' " + + " and sys_imp_date <= '2023-07-07' " + + ") as t_sub_1 " + + "where user_id = '张三' order by play_count desc limit 10", + Collections.singletonList("pv")); Assert.assertTrue(addFieldToSql.toLowerCase().contains("pv")); } @Test void getSqlParseInfo() { - SqlParserInfo sqlParserInfo = SqlParseUtils.getSqlParseInfo( - "select pv from " - + "( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' )" - + " as t_sub_1 "); + SqlParserInfo sqlParserInfo = + SqlParseUtils.getSqlParseInfo( + "select pv from " + + "( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' )" + + " as t_sub_1 "); Assert.assertTrue(sqlParserInfo.getTableName().equalsIgnoreCase("t_1")); - List collect = sqlParserInfo.getAllFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + List collect = + sqlParserInfo.getAllFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(collect.contains("pv")); Assert.assertTrue(!collect.contains("uv")); - List selectFields = sqlParserInfo.getSelectFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + List selectFields = + sqlParserInfo.getSelectFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(selectFields.contains("pv")); Assert.assertTrue(!selectFields.contains("uv")); - sqlParserInfo = SqlParseUtils.getSqlParseInfo( - "select uv from t_1 order by play_count desc limit 10"); + sqlParserInfo = + SqlParseUtils.getSqlParseInfo( + "select uv from t_1 order by play_count desc limit 10"); Assert.assertTrue(sqlParserInfo.getTableName().equalsIgnoreCase("t_1")); - collect = sqlParserInfo.getAllFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + collect = + sqlParserInfo.getAllFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(collect.contains("uv")); Assert.assertTrue(collect.contains("play_count")); Assert.assertTrue(!collect.contains("pv")); - selectFields = sqlParserInfo.getSelectFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + selectFields = + sqlParserInfo.getSelectFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(selectFields.contains("uv")); Assert.assertTrue(!selectFields.contains("pv")); Assert.assertTrue(!selectFields.contains("play_count")); - sqlParserInfo = SqlParseUtils.getSqlParseInfo( - "select uv from " - + "( " - + " select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' " - + ") as t_sub_1 " - + "where user_id = '1' order by play_count desc limit 10" - ); + sqlParserInfo = + SqlParseUtils.getSqlParseInfo( + "select uv from " + + "( " + + " select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' " + + ") as t_sub_1 " + + "where user_id = '1' order by play_count desc limit 10"); Assert.assertTrue(sqlParserInfo.getTableName().equalsIgnoreCase("t_1")); - collect = sqlParserInfo.getAllFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + collect = + sqlParserInfo.getAllFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(collect.contains("uv")); Assert.assertTrue(collect.contains("play_count")); Assert.assertTrue(collect.contains("user_id")); Assert.assertTrue(!collect.contains("pv")); - selectFields = sqlParserInfo.getSelectFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + selectFields = + sqlParserInfo.getSelectFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(selectFields.contains("uv")); Assert.assertTrue(!selectFields.contains("pv")); Assert.assertTrue(!selectFields.contains("user_id")); @@ -117,16 +135,18 @@ class SqlParseUtilsTest { @Test void getWhereFieldTest() { - SqlParserInfo sqlParserInfo = SqlParseUtils.getSqlParseInfo( - "select uv from " - + " ( " - + " select * from t_1 where sys_imp_date >= '2023-07-07' and " - + "sys_imp_date <= '2023-07-07' and user_id = 22 " - + " ) as t_sub_1 " - + " where user_name_元 = 'zhangsan' order by play_count desc limit 10" - ); - List collect = sqlParserInfo.getAllFields().stream().map(field -> field.toLowerCase()) - .collect(Collectors.toList()); + SqlParserInfo sqlParserInfo = + SqlParseUtils.getSqlParseInfo( + "select uv from " + + " ( " + + " select * from t_1 where sys_imp_date >= '2023-07-07' and " + + "sys_imp_date <= '2023-07-07' and user_id = 22 " + + " ) as t_sub_1 " + + " where user_name_元 = 'zhangsan' order by play_count desc limit 10"); + List collect = + sqlParserInfo.getAllFields().stream() + .map(field -> field.toLowerCase()) + .collect(Collectors.toList()); Assert.assertTrue(collect.contains("user_id")); } } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelperTest.java index ccec0c751..962700e16 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelperTest.java @@ -1,5 +1,11 @@ package com.tencent.supersonic.common.jsqlparser; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -7,22 +13,15 @@ import java.util.List; import java.util.Map; import java.util.Set; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import org.junit.Assert; -import org.junit.jupiter.api.Test; - -/** - * SqlParserAddHelperTest Test - */ +/** SqlParserAddHelperTest Test */ class SqlAddHelperTest { @Test void testAddWhere() throws JSQLParserException { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; sql = SqlAddHelper.addWhere(sql, "column_a", 123444555); List selectFields = SqlSelectHelper.getAllSelectFields(sql); @@ -33,93 +32,109 @@ class SqlAddHelperTest { Assert.assertEquals(selectFields.contains("column_b"), true); - Expression expression = CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)"); + Expression expression = + CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)"); - sql = SqlAddHelper.addWhere( - "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1", - expression); + sql = + SqlAddHelper.addWhere( + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1", + expression); Assert.assertEquals(sql.contains("column_c = 111"), true); - sql = "select 部门,sum (访问次数) from 超音数 where 用户 = alice or 发布日期 ='2023-07-03' group by 部门 limit 1"; + sql = + "select 部门,sum (访问次数) from 超音数 where 用户 = alice or 发布日期 ='2023-07-03' group by 部门 limit 1"; sql = SqlAddHelper.addParenthesisToWhere(sql); sql = SqlAddHelper.addWhere(sql, "数据日期", "2023-08-08"); - Assert.assertEquals(sql, "SELECT 部门, sum(访问次数) FROM 超音数 WHERE " - + "(用户 = alice OR 发布日期 = '2023-07-03') AND 数据日期 = '2023-08-08' GROUP BY 部门 LIMIT 1"); - + Assert.assertEquals( + sql, + "SELECT 部门, sum(访问次数) FROM 超音数 WHERE " + + "(用户 = alice OR 发布日期 = '2023-07-03') AND 数据日期 = '2023-08-08' GROUP BY 部门 LIMIT 1"); } @Test void testAddFunctionToSelect() { - String sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + String sql = + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; List havingExpressionList = SqlSelectHelper.getHavingExpression(sql); String replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); - Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + Assert.assertEquals( + "SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", replaceSql); - sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + sql = + "SELECT user_name,sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; havingExpressionList = SqlSelectHelper.getHavingExpression(sql); replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); - Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + Assert.assertEquals( + "SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", replaceSql); - sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') AND " - + "sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + sql = + "SELECT user_name,sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') AND " + + "sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; havingExpressionList = SqlSelectHelper.getHavingExpression(sql); replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); - Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') " + Assert.assertEquals( + "SELECT user_name, sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') " + "AND sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", replaceSql); - } @Test void testAddAggregateToField() { - String sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + String sql = + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; List havingExpressionList = SqlSelectHelper.getHavingExpression(sql); String replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); - Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + Assert.assertEquals( + "SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", replaceSql); - sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + sql = + "SELECT user_name,sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; havingExpressionList = SqlSelectHelper.getHavingExpression(sql); replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); - Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + Assert.assertEquals( + "SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", replaceSql); - sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') AND " - + "sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + sql = + "SELECT user_name,sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') AND " + + "sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; havingExpressionList = SqlSelectHelper.getHavingExpression(sql); replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); - Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') " + Assert.assertEquals( + "SELECT user_name, sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') " + "AND sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", replaceSql); } @Test void testAddAggregateToMetricField() { - String sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' order by pv desc limit 10"; + String sql = + "select department, pv from t_1 where sys_imp_date = '2023-09-11' order by pv desc limit 10"; Map filedNameToAggregate = new HashMap<>(); filedNameToAggregate.put("pv", "sum"); @@ -135,8 +150,9 @@ class SqlAddHelperTest { + "GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 " - + "order by pv desc limit 10"; + sql = + "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 " + + "order by pv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -163,8 +179,9 @@ class SqlAddHelperTest { + "GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' and sum(pv) >1 " - + "GROUP BY department order by pv desc limit 10"; + sql = + "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' and sum(pv) >1 " + + "GROUP BY department order by pv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -173,8 +190,9 @@ class SqlAddHelperTest { + "AND sum(pv) > 1 GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 " - + "GROUP BY department order by pv desc limit 10"; + sql = + "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 " + + "GROUP BY department order by pv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -183,8 +201,9 @@ class SqlAddHelperTest { + "AND sum(pv) > 1 GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 and department = 'HR' " - + "GROUP BY department order by pv desc limit 10"; + sql = + "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 and department = 'HR' " + + "GROUP BY department order by pv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -193,8 +212,9 @@ class SqlAddHelperTest { + "AND department = 'HR' GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, pv from t_1 where (pv >1 and department = 'HR') " - + " and sys_imp_date = '2023-09-11' GROUP BY department order by pv desc limit 10"; + sql = + "select department, pv from t_1 where (pv >1 and department = 'HR') " + + " and sys_imp_date = '2023-09-11' GROUP BY department order by pv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -203,8 +223,9 @@ class SqlAddHelperTest { + "sys_imp_date = '2023-09-11' GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, sum(pv) as pv from t_1 where sys_imp_date = '2023-09-11' GROUP BY " - + "department order by pv desc limit 10"; + sql = + "select department, sum(pv) as pv from t_1 where sys_imp_date = '2023-09-11' GROUP BY " + + "department order by pv desc limit 10"; replaceSql = SqlReplaceHelper.replaceAlias(sql); replaceSql = SqlAddHelper.addAggregateToField(replaceSql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -218,7 +239,8 @@ class SqlAddHelperTest { @Test void testAddAggregateToCountDiscountMetricField() { - String sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' order by uv desc limit 10"; + String sql = + "select department, uv from t_1 where sys_imp_date = '2023-09-11' order by uv desc limit 10"; Map filedNameToAggregate = new HashMap<>(); filedNameToAggregate.put("uv", "count_distinct"); @@ -234,8 +256,9 @@ class SqlAddHelperTest { + "GROUP BY department ORDER BY count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 " - + "order by uv desc limit 10"; + sql = + "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 " + + "order by uv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -253,7 +276,8 @@ class SqlAddHelperTest { + "GROUP BY department ORDER BY count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, uv from t_1 where count(DISTINCT uv) >1 order by uv desc limit 10"; + sql = + "select department, uv from t_1 where count(DISTINCT uv) >1 order by uv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -262,9 +286,10 @@ class SqlAddHelperTest { + "GROUP BY department ORDER BY count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, count(DISTINCT uv) from t_1 where sys_imp_date = '2023-09-11'" - + " and count(DISTINCT uv) >1 " - + "GROUP BY department order by count(DISTINCT uv) desc limit 10"; + sql = + "select department, count(DISTINCT uv) from t_1 where sys_imp_date = '2023-09-11'" + + " and count(DISTINCT uv) >1 " + + "GROUP BY department order by count(DISTINCT uv) desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -273,8 +298,9 @@ class SqlAddHelperTest { + "AND count(DISTINCT uv) > 1 GROUP BY department ORDER BY count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 " - + "GROUP BY department order by count(DISTINCT uv) desc limit 10"; + sql = + "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 " + + "GROUP BY department order by count(DISTINCT uv) desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -283,8 +309,9 @@ class SqlAddHelperTest { + "AND count(DISTINCT uv) > 1 GROUP BY department ORDER BY count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 and department = 'HR' " - + "GROUP BY department order by uv desc limit 10"; + sql = + "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 and department = 'HR' " + + "GROUP BY department order by uv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -294,8 +321,9 @@ class SqlAddHelperTest { + "AND department = 'HR' GROUP BY department ORDER BY count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, uv from t_1 where (uv >1 and department = 'HR') " - + " and sys_imp_date = '2023-09-11' GROUP BY department order by uv desc limit 10"; + sql = + "select department, uv from t_1 where (uv >1 and department = 'HR') " + + " and sys_imp_date = '2023-09-11' GROUP BY department order by uv desc limit 10"; replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -306,8 +334,9 @@ class SqlAddHelperTest { + "count(DISTINCT uv) DESC LIMIT 10", replaceSql); - sql = "select department, count(DISTINCT uv) as uv from t_1 where sys_imp_date = '2023-09-11' GROUP BY " - + "department order by uv desc limit 10"; + sql = + "select department, count(DISTINCT uv) as uv from t_1 where sys_imp_date = '2023-09-11' GROUP BY " + + "department order by uv desc limit 10"; replaceSql = SqlReplaceHelper.replaceAlias(sql); replaceSql = SqlAddHelper.addAggregateToField(replaceSql, filedNameToAggregate); replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); @@ -321,8 +350,9 @@ class SqlAddHelperTest { @Test void testAddGroupBy() { - String sql = "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' " - + "order by sum(pv) desc limit 10"; + String sql = + "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' " + + "order by sum(pv) desc limit 10"; Set groupByFields = new HashSet<>(); groupByFields.add("department"); @@ -334,8 +364,9 @@ class SqlAddHelperTest { + "GROUP BY department ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, sum(pv) from t_1 where (department = 'HR') and sys_imp_date = '2023-09-11' " - + "order by sum(pv) desc limit 10"; + sql = + "select department, sum(pv) from t_1 where (department = 'HR') and sys_imp_date = '2023-09-11' " + + "order by sum(pv) desc limit 10"; replaceSql = SqlAddHelper.addGroupBy(sql, groupByFields); @@ -347,8 +378,9 @@ class SqlAddHelperTest { @Test void testAddHaving() { - String sql = "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' and " - + "sum(pv) > 2000 group by department order by sum(pv) desc limit 10"; + String sql = + "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' and " + + "sum(pv) > 2000 group by department order by sum(pv) desc limit 10"; List groupByFields = new ArrayList<>(); groupByFields.add("department"); @@ -362,8 +394,9 @@ class SqlAddHelperTest { + "GROUP BY department HAVING sum(pv) > 2000 ORDER BY sum(pv) DESC LIMIT 10", replaceSql); - sql = "select department, sum(pv) from t_1 where (sum(pv) > 2000) and sys_imp_date = '2023-09-11' " - + "group by department order by sum(pv) desc limit 10"; + sql = + "select department, sum(pv) from t_1 where (sum(pv) > 2000) and sys_imp_date = '2023-09-11' " + + "group by department order by sum(pv) desc limit 10"; replaceSql = SqlAddHelper.addHaving(sql, fieldNames); @@ -375,9 +408,10 @@ class SqlAddHelperTest { @Test void testAddParenthesisToWhere() { - String sql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + String sql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; String replaceSql = SqlAddHelper.addParenthesisToWhere(sql); @@ -391,11 +425,11 @@ class SqlAddHelperTest { @Test void testAddFieldsToSelect() { String correctS2SQL = "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC"; - String replaceFields = SqlAddHelper.addFieldsToSelect(correctS2SQL, - SqlSelectHelper.getOrderByFields(correctS2SQL)); + String replaceFields = + SqlAddHelper.addFieldsToSelect( + correctS2SQL, SqlSelectHelper.getOrderByFields(correctS2SQL)); Assert.assertEquals( "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC", replaceFields); } - } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java index d1ed5960f..c5ae88741 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; - import com.tencent.supersonic.common.jsqlparser.DateVisitor.DateBoundInfo; import org.junit.Assert; import org.junit.jupiter.api.Test; @@ -10,37 +9,42 @@ class SqlDateSelectHelperTest { @Test void testGetDateBoundInfo() { - String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; + String sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); Assert.assertEquals(dateBoundInfo.getLowerBound(), ">="); Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17"); - sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; + sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); Assert.assertEquals(dateBoundInfo.getLowerBound(), ">"); Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17"); - sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); Assert.assertEquals(dateBoundInfo.getUpperBound(), "<="); Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); - sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; + sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); Assert.assertEquals(dateBoundInfo.getUpperBound(), "<"); Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); - sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-10-17' " - + "AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-10-17' " + + "AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); Assert.assertEquals(dateBoundInfo.getUpperBound(), "<="); Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); Assert.assertEquals(dateBoundInfo.getLowerBound(), ">="); Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-10-17"); } -} \ No newline at end of file +} diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java index 09e859214..700684a1b 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java @@ -7,9 +7,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -/** - * SqlParser Remove Helper Test - */ +/** SqlParser Remove Helper Test */ class SqlRemoveHelperTest { @Test @@ -28,40 +26,46 @@ class SqlRemoveHelperTest { @Test void testRemoveSameFieldFromSelect() { - String sql = "select 歌曲名,歌手名,粉丝数,粉丝数,sum(粉丝数),sum(粉丝数),avg(播放量),avg(播放量)" - + " from 歌曲库 where sum(粉丝数) > 20000 and 2>1 and " - + "sum(播放量) > 20000 and 1=1 HAVING sum(播放量) > 20000 and 3>1"; + String sql = + "select 歌曲名,歌手名,粉丝数,粉丝数,sum(粉丝数),sum(粉丝数),avg(播放量),avg(播放量)" + + " from 歌曲库 where sum(粉丝数) > 20000 and 2>1 and " + + "sum(播放量) > 20000 and 1=1 HAVING sum(播放量) > 20000 and 3>1"; sql = SqlRemoveHelper.removeSameFieldFromSelect(sql); System.out.println(sql); - sql = "SELECT 结算播放量 FROM 艺人 WHERE (歌手名 IN ('林俊杰', '陈奕迅')) AND (数据日期 >= '2024-04-04' AND 数据日期 <= '2024-04-04')"; + sql = + "SELECT 结算播放量 FROM 艺人 WHERE (歌手名 IN ('林俊杰', '陈奕迅')) AND (数据日期 >= '2024-04-04' AND 数据日期 <= '2024-04-04')"; List fieldExpressionList = SqlSelectHelper.getWhereExpressions(sql); - fieldExpressionList.stream().forEach(fieldExpression -> { - System.out.println(fieldExpression.toString()); - }); - + fieldExpressionList.stream() + .forEach( + fieldExpression -> { + System.out.println(fieldExpression.toString()); + }); } @Test void testRemoveWhereHavingCondition() { - String sql = "select 歌曲名 from 歌曲库 where sum(粉丝数) > 20000 and 2>1 and " - + "sum(播放量) > 20000 and 1=1 HAVING sum(播放量) > 20000 and 3>1"; + String sql = + "select 歌曲名 from 歌曲库 where sum(粉丝数) > 20000 and 2>1 and " + + "sum(播放量) > 20000 and 1=1 HAVING sum(播放量) > 20000 and 3>1"; sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE sum(粉丝数) > 20000 AND sum(播放量) > 20000 HAVING sum(播放量) > 20000", sql); - sql = "SELECT 歌曲,sum(播放量) FROM 歌曲库\n" - + "WHERE (歌手名 = '张三' AND 2 > 1) AND 数据日期 = '2023-11-07'\n" - + "GROUP BY 歌曲名 HAVING sum(播放量) > 100000"; + sql = + "SELECT 歌曲,sum(播放量) FROM 歌曲库\n" + + "WHERE (歌手名 = '张三' AND 2 > 1) AND 数据日期 = '2023-11-07'\n" + + "GROUP BY 歌曲名 HAVING sum(播放量) > 100000"; sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( "SELECT 歌曲, sum(播放量) FROM 歌曲库 WHERE (歌手名 = '张三') " + "AND 数据日期 = '2023-11-07' GROUP BY 歌曲名 HAVING sum(播放量) > 100000", sql); - sql = "SELECT 歌曲名,sum(播放量) FROM 歌曲库 WHERE (1 = 1 AND 1 = 1 AND 2 > 1 )" - + "AND 1 = 1 AND 歌曲类型 IN ('类型一', '类型二') AND 歌手名 IN ('林俊杰', '周杰伦')" - + "AND 数据日期 = '2023-11-07' GROUP BY 歌曲名 HAVING 2 > 1 AND SUM(播放量) >= 1000"; + sql = + "SELECT 歌曲名,sum(播放量) FROM 歌曲库 WHERE (1 = 1 AND 1 = 1 AND 2 > 1 )" + + "AND 1 = 1 AND 歌曲类型 IN ('类型一', '类型二') AND 歌手名 IN ('林俊杰', '周杰伦')" + + "AND 数据日期 = '2023-11-07' GROUP BY 歌曲名 HAVING 2 > 1 AND SUM(播放量) >= 1000"; sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( @@ -70,8 +74,9 @@ class SqlRemoveHelperTest { + "GROUP BY 歌曲名 HAVING SUM(播放量) >= 1000", sql); - sql = "SELECT 品牌名称,法人 FROM 互联网企业 WHERE (2 > 1 AND 1 = 1) AND 数据日期 = '2023-10-31'" - + "GROUP BY 品牌名称, 法人 HAVING 2 > 1 AND sum(注册资本) > 100000000 AND sum(营收占比) = 0.5 and 1 = 1"; + sql = + "SELECT 品牌名称,法人 FROM 互联网企业 WHERE (2 > 1 AND 1 = 1) AND 数据日期 = '2023-10-31'" + + "GROUP BY 品牌名称, 法人 HAVING 2 > 1 AND sum(注册资本) > 100000000 AND sum(营收占比) = 0.5 and 1 = 1"; sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( @@ -86,17 +91,15 @@ class SqlRemoveHelperTest { Set removeFieldNames = new HashSet<>(); removeFieldNames.add("播放量"); String replaceSql = SqlRemoveHelper.removeHavingCondition(sql, removeFieldNames); - Assert.assertEquals( - "SELECT 歌曲名 FROM 歌曲库 WHERE 歌手名 = '周杰伦'", - replaceSql); - + Assert.assertEquals("SELECT 歌曲名 FROM 歌曲库 WHERE 歌手名 = '周杰伦'", replaceSql); } @Test void testRemoveWhereCondition() { - String sql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + String sql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; Set removeFieldNames = new HashSet<>(); removeFieldNames.add("歌曲名"); @@ -109,9 +112,10 @@ class SqlRemoveHelperTest { + "ORDER BY 播放量 DESC LIMIT 11", replaceSql); - sql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌曲名 in ('邓紫棋','周杰伦') and 歌曲名 in ('邓紫棋') and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + sql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌曲名 in ('邓紫棋','周杰伦') and 歌曲名 in ('邓紫棋') and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; replaceSql = SqlRemoveHelper.removeWhereCondition(sql, removeFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " @@ -119,9 +123,10 @@ class SqlRemoveHelperTest { + "歌曲发布时 = '2023-08-01' ORDER BY 播放量 DESC LIMIT 11", replaceSql); - sql = "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌曲名 in ('邓紫棋','周杰伦') and 歌曲名 in ('邓紫棋')) and 数据日期 = '2023-08-09' " - + " order by 播放量 desc limit 11"; + sql = + "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌曲名 in ('邓紫棋','周杰伦') and 歌曲名 in ('邓紫棋')) and 数据日期 = '2023-08-09' " + + " order by 播放量 desc limit 11"; replaceSql = SqlRemoveHelper.removeWhereCondition(sql, removeFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1) " @@ -131,7 +136,8 @@ class SqlRemoveHelperTest { @Test void testRemoveSelect() { - String sql = "select 数据日期,歌曲名 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时间 = '2023-08-01'"; + String sql = + "select 数据日期,歌曲名 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时间 = '2023-08-01'"; Set removeFieldNames = new HashSet<>(); removeFieldNames.add("数据日期"); @@ -141,7 +147,8 @@ class SqlRemoveHelperTest { "SELECT 歌曲名 FROM 歌曲库 WHERE 歌曲名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01'", replaceSql); - sql = "select 数据日期 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时间 = '2023-08-01'"; + sql = + "select 数据日期 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时间 = '2023-08-01'"; replaceSql = SqlRemoveHelper.removeSelect(sql, removeFieldNames); @@ -152,8 +159,9 @@ class SqlRemoveHelperTest { @Test void testRemoveGroupBy() { - String sql = "select 数据日期 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and " - + "歌曲发布时间 = '2023-08-01' group by 数据日期"; + String sql = + "select 数据日期 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and " + + "歌曲发布时间 = '2023-08-01' group by 数据日期"; Set removeFieldNames = new HashSet<>(); removeFieldNames.add("数据日期"); diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java index d527ecf8c..1e9d228d8 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java @@ -11,50 +11,58 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -/** - * SqlParserReplaceHelperTest - */ +/** SqlParserReplaceHelperTest */ class SqlReplaceHelperTest { @Test void testReplaceSelectField() { - String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; + String sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; Map fieldMap = new HashMap<>(); fieldMap.put("播放量", "播放量1"); sql = SqlReplaceHelper.replaceSelectFields(sql, fieldMap); System.out.println(sql); - Assert.assertEquals("SELECT 维度1, sum(播放量1) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", sql); + Assert.assertEquals( + "SELECT 维度1, sum(播放量1) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", + sql); - sql = "SELECT 维度1,播放量 FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; + sql = + "SELECT 维度1,播放量 FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; fieldMap = new HashMap<>(); fieldMap.put("播放量", "播放量1"); sql = SqlReplaceHelper.replaceSelectFields(sql, fieldMap); System.out.println(sql); - Assert.assertEquals("SELECT 维度1, 播放量1 FROM 数据库 WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", sql); + Assert.assertEquals( + "SELECT 维度1, 播放量1 FROM 数据库 WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", + sql); } @Test void testReplaceAggField() { - String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; + String sql = + "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; Map> fieldMap = new HashMap<>(); fieldMap.put("播放量", Pair.of("收听用户数", AggOperatorEnum.COUNT_DISTINCT.name())); sql = SqlReplaceHelper.replaceAggFields(sql, fieldMap); System.out.println(sql); - Assert.assertEquals("SELECT 维度1, count(DISTINCT 收听用户数) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", sql); + Assert.assertEquals( + "SELECT 维度1, count(DISTINCT 收听用户数) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", + sql); } @Test void testReplaceValue() { - String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '杰伦' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + String replaceSql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '杰伦' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; Map> filedNameToValueMap = new HashMap<>(); @@ -67,11 +75,13 @@ class SqlReplaceHelperTest { Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND " + "歌手名 = '周杰伦' AND 数据日期 = '2023-08-09' AND 歌曲发布时 = '2023-08-01' " - + "ORDER BY 播放量 DESC LIMIT 11", replaceSql); + + "ORDER BY 播放量 DESC LIMIT 11", + replaceSql); - replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + replaceSql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; Map> filedNameToValueMap2 = new HashMap<>(); @@ -86,24 +96,28 @@ class SqlReplaceHelperTest { Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '周杰伦' " + "AND 歌手名 = '林俊杰' AND 歌手名 = '陈奕迅' AND 数据日期 = '2023-08-09' AND " - + "歌曲发布时 = '2023-08-01' ORDER BY 播放量 DESC LIMIT 11", replaceSql); + + "歌曲发布时 = '2023-08-01' ORDER BY 播放量 DESC LIMIT 11", + replaceSql); - replaceSql = "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 歌曲发布时 = '2023-08-01') and 数据日期 = '2023-08-09' " - + " order by 播放量 desc limit 11"; + replaceSql = + "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 歌曲发布时 = '2023-08-01') and 数据日期 = '2023-08-09' " + + " order by 播放量 desc limit 11"; replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '周杰伦' " + "AND 歌手名 = '林俊杰' AND 歌手名 = '陈奕迅' AND 歌曲发布时 = '2023-08-01') " - + "AND 数据日期 = '2023-08-09' ORDER BY 播放量 DESC LIMIT 11", replaceSql); + + "AND 数据日期 = '2023-08-09' ORDER BY 播放量 DESC LIMIT 11", + replaceSql); - replaceSql = "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 歌曲发布时 = '2023-08-01' and 播放量 < (" - + "select min(播放量) from 歌曲库 where 语种 = '英文' " - + ") ) and 数据日期 = '2023-08-09' " - + " order by 播放量 desc limit 11"; + replaceSql = + "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 歌曲发布时 = '2023-08-01' and 播放量 < (" + + "select min(播放量) from 歌曲库 where 语种 = '英文' " + + ") ) and 数据日期 = '2023-08-09' " + + " order by 播放量 desc limit 11"; replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); @@ -111,8 +125,8 @@ class SqlReplaceHelperTest { "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 AND " + "歌手名 = '周杰伦' AND 歌手名 = '林俊杰' AND 歌手名 = '陈奕迅' AND 歌曲发布时 = '2023-08-01' " + "AND 播放量 < (SELECT min(播放量) FROM 歌曲库 WHERE 语种 = '英文')) AND 数据日期 = '2023-08-09' " - + "ORDER BY 播放量 DESC LIMIT 11", replaceSql); - + + "ORDER BY 播放量 DESC LIMIT 11", + replaceSql); Map> filedNameToValueMap3 = new HashMap<>(); @@ -124,16 +138,16 @@ class SqlReplaceHelperTest { replaceSql = "SELECT 歌曲名 FROM 歌曲库 WHERE 歌手名 in ('周杰伦','林俊杰','陈奕迅') "; replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap3, true); - Assert.assertEquals( - "SELECT 歌曲名 FROM 歌曲库 WHERE 歌手名 IN ('1', '2', '3')", replaceSql); + Assert.assertEquals("SELECT 歌曲名 FROM 歌曲库 WHERE 歌手名 IN ('1', '2', '3')", replaceSql); } @Test void testReplaceFieldNameByValue() { - String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + String replaceSql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; Map> fieldValueToFieldNames = new HashMap<>(); fieldValueToFieldNames.put("邓紫棋", Collections.singleton("歌手名")); @@ -143,18 +157,21 @@ class SqlReplaceHelperTest { Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND " + "歌手名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时 = '2023-08-01' " - + "ORDER BY 播放量 DESC LIMIT 11", replaceSql); + + "ORDER BY 播放量 DESC LIMIT 11", + replaceSql); - replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌曲名 like '%邓紫棋%' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + replaceSql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌曲名 like '%邓紫棋%' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " + "AND 歌曲名 LIKE '%邓紫棋%' AND 数据日期 = '2023-08-09' AND 歌曲发布时 = " - + "'2023-08-01' ORDER BY 播放量 DESC LIMIT 11", replaceSql); + + "'2023-08-01' ORDER BY 播放量 DESC LIMIT 11", + replaceSql); Set fieldNames = new HashSet<>(); fieldNames.add("歌手名"); @@ -162,28 +179,31 @@ class SqlReplaceHelperTest { fieldNames.add("专辑名"); fieldValueToFieldNames.put("林俊杰", fieldNames); - replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '林俊杰' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + replaceSql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '林俊杰' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " + "AND 歌手名 = '林俊杰' AND 数据日期 = '2023-08-09' AND 歌曲发布时 = " - + "'2023-08-01' ORDER BY 播放量 DESC LIMIT 11", replaceSql); + + "'2023-08-01' ORDER BY 播放量 DESC LIMIT 11", + replaceSql); - replaceSql = "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '林俊杰' and 歌曲发布时 = '2023-08-01') and 数据日期 = '2023-08-09'" - + " order by 播放量 desc limit 11"; + replaceSql = + "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '林俊杰' and 歌曲发布时 = '2023-08-01') and 数据日期 = '2023-08-09'" + + " order by 播放量 desc limit 11"; replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 AND " + "歌手名 = '林俊杰' AND 歌曲发布时 = '2023-08-01') AND 数据日期 = '2023-08-09' " - + "ORDER BY 播放量 DESC LIMIT 11", replaceSql); - + + "ORDER BY 播放量 DESC LIMIT 11", + replaceSql); } @Test @@ -191,39 +211,46 @@ class SqlReplaceHelperTest { Map fieldToBizName1 = new HashMap<>(); fieldToBizName1.put("公司成立时间", "company_established_time"); fieldToBizName1.put("年营业额", "annual_turnover"); - String replaceSql = "SELECT * FROM 互联网企业 ORDER BY 公司成立时间 DESC LIMIT 3 " - + "UNION SELECT * FROM 互联网企业 ORDER BY 年营业额 DESC LIMIT 5"; + String replaceSql = + "SELECT * FROM 互联网企业 ORDER BY 公司成立时间 DESC LIMIT 3 " + + "UNION SELECT * FROM 互联网企业 ORDER BY 年营业额 DESC LIMIT 5"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName1); replaceSql = SqlReplaceHelper.replaceTable(replaceSql, "internet"); Assert.assertEquals( "SELECT * FROM internet ORDER BY company_established_time DESC LIMIT 3 " - + "UNION SELECT * FROM internet ORDER BY annual_turnover DESC LIMIT 5", replaceSql); + + "UNION SELECT * FROM internet ORDER BY annual_turnover DESC LIMIT 5", + replaceSql); } @Test void testReplaceFields() { Map fieldToBizName = initParams(); - String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" - + " order by 播放量 desc limit 11"; + String replaceSql = + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + + " order by 播放量 desc limit 11"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE (publish_date >= '2023-08-08' AND publish_date <= '2023-08-09')" + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND song_publis_date = '2023-08-01'" - + " ORDER BY play_count DESC LIMIT 11", replaceSql); + + " ORDER BY play_count DESC LIMIT 11", + replaceSql); - replaceSql = "select 品牌名称 from 互联网企业 where datediff('year', 品牌成立时间, '2023-11-04') > 17 and 注册资本 = 50000000"; + replaceSql = + "select 品牌名称 from 互联网企业 where datediff('year', 品牌成立时间, '2023-11-04') > 17 and 注册资本 = 50000000"; replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); replaceSql = SqlRemoveHelper.removeNumberFilter(replaceSql); Assert.assertEquals( - "SELECT 品牌名称 FROM 互联网企业 WHERE 品牌成立时间 < '2006-11-04' AND 注册资本 = 50000000", replaceSql); + "SELECT 品牌名称 FROM 互联网企业 WHERE 品牌成立时间 < '2006-11-04' AND 注册资本 = 50000000", + replaceSql); - replaceSql = "select MONTH(数据日期), sum(访问次数) from 内容库产品 " - + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " - + "group by MONTH(数据日期) order by sum(访问次数) desc limit 1"; + replaceSql = + "select MONTH(数据日期), sum(访问次数) from 内容库产品 " + + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " + + "group by MONTH(数据日期) order by sum(访问次数) desc limit 1"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); @@ -231,11 +258,13 @@ class SqlReplaceHelperTest { Assert.assertEquals( "SELECT MONTH(sys_imp_date), sum(pv) FROM 内容库产品 WHERE (sys_imp_date >= '2023-03-03' " + "AND sys_imp_date <= '2023-09-03')" - + " GROUP BY MONTH(sys_imp_date) ORDER BY sum(pv) DESC LIMIT 1", replaceSql); + + " GROUP BY MONTH(sys_imp_date) ORDER BY sum(pv) DESC LIMIT 1", + replaceSql); - replaceSql = "select MONTH(数据日期), sum(访问次数) from 内容库产品 " - + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " - + "group by MONTH(数据日期) HAVING sum(访问次数) > 1000"; + replaceSql = + "select MONTH(数据日期), sum(访问次数) from 内容库产品 " + + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " + + "group by MONTH(数据日期) HAVING sum(访问次数) > 1000"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); @@ -245,8 +274,9 @@ class SqlReplaceHelperTest { + " sys_imp_date <= '2023-09-03') GROUP BY MONTH(sys_imp_date) HAVING sum(pv) > 1000", replaceSql); - replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) " - + "in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)"; + replaceSql = + "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) " + + "in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); @@ -257,9 +287,10 @@ class SqlReplaceHelperTest { + "GROUP BY YEAR(publish_date)", replaceSql); - replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 " - + "where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' " - + "group by 发行日期"; + replaceSql = + "select YEAR(发行日期), count(歌曲名) from 歌曲库 " + + "where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' " + + "group by 发行日期"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); @@ -270,10 +301,11 @@ class SqlReplaceHelperTest { + " GROUP BY publish_date", replaceSql); - replaceSql = SqlReplaceHelper.replaceFields( - "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-11') <= 1 " - + "and 结算播放量 > 1000000 and datediff('day', 数据日期, '2023-08-11') <= 30", - fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-11') <= 1 " + + "and 结算播放量 > 1000000 and datediff('day', 数据日期, '2023-08-11') <= 30", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( @@ -282,10 +314,11 @@ class SqlReplaceHelperTest { + "(sys_imp_date >= '2023-07-12' AND sys_imp_date <= '2023-08-11')", replaceSql); - replaceSql = SqlReplaceHelper.replaceFields( - "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " - + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", - fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( @@ -293,9 +326,11 @@ class SqlReplaceHelperTest { + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlReplaceHelper.replaceFields( - "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') = 0 " - + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') = 0 " + + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( @@ -303,9 +338,11 @@ class SqlReplaceHelperTest { + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlReplaceHelper.replaceFields( - "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 " - + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 " + + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( @@ -313,108 +350,133 @@ class SqlReplaceHelperTest { + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlReplaceHelper.replaceFields( - "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 " - + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 " + + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); replaceSql = SqlRemoveHelper.removeNumberFilter(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-02-09' AND" + " singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09'" - + " ORDER BY play_count DESC LIMIT 11", replaceSql); + + " ORDER BY play_count DESC LIMIT 11", + replaceSql); - replaceSql = SqlReplaceHelper.replaceFields( - "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice'" - + " and 发布日期 ='11' order by 访问次数 desc limit 1", fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice'" + + " and 发布日期 ='11' order by 访问次数 desc limit 1", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT department, user_id FROM 超音数 WHERE sys_imp_date = '2023-08-08'" - + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1", replaceSql); + + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1", + replaceSql); replaceSql = SqlReplaceHelper.replaceTable(replaceSql, "s2"); - replaceSql = SqlAddHelper.addFieldsToSelect(replaceSql, Collections.singletonList("field_a")); + replaceSql = + SqlAddHelper.addFieldsToSelect(replaceSql, Collections.singletonList("field_a")); - replaceSql = SqlReplaceHelper.replaceFields( - "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 ='alice' and 发布日期 ='11' group by 部门 limit 1", fieldToBizName); + replaceSql = + SqlReplaceHelper.replaceFields( + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 ='alice' and 发布日期 ='11' group by 部门 limit 1", + fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08'" - + " AND user_id = 'alice' AND publish_date = '11' GROUP BY department LIMIT 1", replaceSql); + + " AND user_id = 'alice' AND publish_date = '11' GROUP BY department LIMIT 1", + replaceSql); - replaceSql = "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' " - + "and 数据日期 <= '2023-08-06' and 部门 = 'hr'"; + replaceSql = + "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' " + + "and 数据日期 <= '2023-08-06' and 部门 = 'hr'"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT sum(pv) FROM 超音数 WHERE sys_imp_date >= '2023-08-06' " - + "AND sys_imp_date <= '2023-08-06' AND department = 'hr'", replaceSql); + + "AND sys_imp_date <= '2023-08-06' AND department = 'hr'", + replaceSql); - replaceSql = "SELECT 歌曲名称, sum(评分) FROM CSpider WHERE(1 < 2) AND 数据日期 = '2023-10-15' " - + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider WHERE 语种 = '英文')"; + replaceSql = + "SELECT 歌曲名称, sum(评分) FROM CSpider WHERE(1 < 2) AND 数据日期 = '2023-10-15' " + + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider WHERE 语种 = '英文')"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name, sum(评分) FROM CSpider WHERE (1 < 2) AND " + "sys_imp_date = '2023-10-15' GROUP BY song_name HAVING " - + "sum(评分) < (SELECT min(评分) FROM CSpider WHERE user_id = '英文')", replaceSql); + + "sum(评分) < (SELECT min(评分) FROM CSpider WHERE user_id = '英文')", + replaceSql); - replaceSql = "SELECT sum(评分)/ (SELECT sum(评分) FROM CSpider WHERE 数据日期 = '2023-10-15')" - + " FROM CSpider WHERE 数据日期 = '2023-10-15' " - + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider WHERE 语种 = '英文')"; + replaceSql = + "SELECT sum(评分)/ (SELECT sum(评分) FROM CSpider WHERE 数据日期 = '2023-10-15')" + + " FROM CSpider WHERE 数据日期 = '2023-10-15' " + + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider WHERE 语种 = '英文')"; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT sum(评分) / (SELECT sum(评分) FROM CSpider WHERE sys_imp_date = '2023-10-15') " + "FROM CSpider WHERE sys_imp_date = '2023-10-15' GROUP BY song_name HAVING " - + "sum(评分) < (SELECT min(评分) FROM CSpider WHERE user_id = '英文')", replaceSql); + + "sum(评分) < (SELECT min(评分) FROM CSpider WHERE user_id = '英文')", + replaceSql); } @Test void testReplaceFunctionField() { Map fieldToBizName = initParams(); - String replaceSql = "SELECT TIMESTAMPDIFF (MONTH,歌曲发布时间,CURDATE()) AS 发布月数 FROM 歌曲库 WHERE 歌手名 = '邓紫棋' "; + String replaceSql = + "SELECT TIMESTAMPDIFF (MONTH,歌曲发布时间,CURDATE()) AS 发布月数 FROM 歌曲库 WHERE 歌手名 = '邓紫棋' "; replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT TIMESTAMPDIFF(MONTH, song_publis_date, CURDATE()) AS 发布月数 " - + "FROM 歌曲库 WHERE singer_name = '邓紫棋'", replaceSql); + + "FROM 歌曲库 WHERE singer_name = '邓紫棋'", + replaceSql); } @Test void testReplaceTable() { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; String replaceSql = SqlReplaceHelper.replaceTable(sql, "s2"); Assert.assertEquals( "SELECT 部门, sum(访问次数) FROM s2 WHERE 数据日期 = '2023-08-08' " - + "AND 用户 = alice AND 发布日期 = '11' GROUP BY 部门 LIMIT 1", replaceSql); + + "AND 用户 = alice AND 发布日期 = '11' GROUP BY 部门 LIMIT 1", + replaceSql); - sql = "select * from 互联网企业 order by 公司成立时间 desc limit 3 union select * from 互联网企业 order by 年营业额 desc limit 5"; + sql = + "select * from 互联网企业 order by 公司成立时间 desc limit 3 union select * from 互联网企业 order by 年营业额 desc limit 5"; replaceSql = SqlReplaceHelper.replaceTable(sql, "internet"); Assert.assertEquals( "SELECT * FROM internet ORDER BY 公司成立时间 DESC LIMIT 3 " - + "UNION SELECT * FROM internet ORDER BY 年营业额 DESC LIMIT 5", replaceSql); + + "UNION SELECT * FROM internet ORDER BY 年营业额 DESC LIMIT 5", + replaceSql); - sql = "SELECT * FROM CSpider音乐 WHERE (评分 < (SELECT min(评分) " - + "FROM CSpider音乐 WHERE 语种 = '英文')) AND 数据日期 = '2023-10-11'"; + sql = + "SELECT * FROM CSpider音乐 WHERE (评分 < (SELECT min(评分) " + + "FROM CSpider音乐 WHERE 语种 = '英文')) AND 数据日期 = '2023-10-11'"; replaceSql = SqlReplaceHelper.replaceTable(sql, "cspider"); Assert.assertEquals( "SELECT * FROM cspider WHERE (评分 < (SELECT min(评分) FROM " - + "cspider WHERE 语种 = '英文')) AND 数据日期 = '2023-10-11'", replaceSql); + + "cspider WHERE 语种 = '英文')) AND 数据日期 = '2023-10-11'", + replaceSql); - sql = "SELECT 歌曲名称, sum(评分) FROM CSpider音乐 WHERE(1 < 2) AND 数据日期 = '2023-10-15' " - + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider音乐 WHERE 语种 = '英文')"; + sql = + "SELECT 歌曲名称, sum(评分) FROM CSpider音乐 WHERE(1 < 2) AND 数据日期 = '2023-10-15' " + + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider音乐 WHERE 语种 = '英文')"; replaceSql = SqlReplaceHelper.replaceTable(sql, "cspider"); @@ -428,20 +490,24 @@ class SqlReplaceHelperTest { @Test void testReplaceFunctionName() { - String sql = "select 公司名称,平均(注册资本),总部地点 from 互联网企业 where\n" - + "年营业额 >= 28800000000 and 最大(注册资本)>10000 \n" - + " group by 公司名称 having 平均(注册资本)>10000 order by \n" - + "平均(注册资本) desc limit 5"; + String sql = + "select 公司名称,平均(注册资本),总部地点 from 互联网企业 where\n" + + "年营业额 >= 28800000000 and 最大(注册资本)>10000 \n" + + " group by 公司名称 having 平均(注册资本)>10000 order by \n" + + "平均(注册资本) desc limit 5"; Map map = new HashMap<>(); map.put("平均", "avg"); map.put("最大", "max"); sql = SqlReplaceHelper.replaceFunction(sql, map); System.out.println(sql); - Assert.assertEquals("SELECT 公司名称, avg(注册资本), 总部地点 FROM 互联网企业 WHERE 年营业额 >= 28800000000 AND " - + "max(注册资本) > 10000 GROUP BY 公司名称 HAVING avg(注册资本) > 10000 ORDER BY avg(注册资本) DESC LIMIT 5", sql); + Assert.assertEquals( + "SELECT 公司名称, avg(注册资本), 总部地点 FROM 互联网企业 WHERE 年营业额 >= 28800000000 AND " + + "max(注册资本) > 10000 GROUP BY 公司名称 HAVING avg(注册资本) > 10000 ORDER BY avg(注册资本) DESC LIMIT 5", + sql); - sql = "select MONTH(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" - + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; + sql = + "select MONTH(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; Map functionMap = new HashMap<>(); functionMap.put("MONTH".toLowerCase(), "toMonth"); String replaceSql = SqlReplaceHelper.replaceFunction(sql, functionMap); @@ -451,8 +517,9 @@ class SqlReplaceHelperTest { + " datediff('month', 数据日期, '2023-09-02') <= 6 GROUP BY toMonth(数据日期)", replaceSql); - sql = "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" - + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; + sql = + "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; replaceSql = SqlReplaceHelper.replaceFunction(sql, functionMap); Assert.assertEquals( @@ -460,8 +527,9 @@ class SqlReplaceHelperTest { + " datediff('month', 数据日期, '2023-09-02') <= 6 GROUP BY toMonth(数据日期)", replaceSql); - sql = "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" - + " (datediff('month', 数据日期, '2023-09-02') <= 6) and 数据日期 = '2023-10-10' group by MONTH(数据日期)"; + sql = + "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + + " (datediff('month', 数据日期, '2023-09-02') <= 6) and 数据日期 = '2023-10-10' group by MONTH(数据日期)"; replaceSql = SqlReplaceHelper.replaceFunction(sql, functionMap); Assert.assertEquals( @@ -473,8 +541,9 @@ class SqlReplaceHelperTest { @Test void testReplaceAlias() { - String sql = "select 部门, sum(访问次数) as 总访问次数 from 超音数 where " - + "datediff('day', 数据日期, '2023-09-05') <= 3 group by 部门 order by 总访问次数 desc limit 10"; + String sql = + "select 部门, sum(访问次数) as 总访问次数 from 超音数 where " + + "datediff('day', 数据日期, '2023-09-05') <= 3 group by 部门 order by 总访问次数 desc limit 10"; String replaceSql = SqlReplaceHelper.replaceAlias(sql); System.out.println(replaceSql); Assert.assertEquals( @@ -482,9 +551,10 @@ class SqlReplaceHelperTest { + "datediff('day', 数据日期, '2023-09-05') <= 3 GROUP BY 部门 ORDER BY sum(访问次数) DESC LIMIT 10", replaceSql); - sql = "select 部门, sum(访问次数) as 总访问次数 from 超音数 where " - + "(datediff('day', 数据日期, '2023-09-05') <= 3) and 数据日期 = '2023-10-10' " - + "group by 部门 order by 总访问次数 desc limit 10"; + sql = + "select 部门, sum(访问次数) as 总访问次数 from 超音数 where " + + "(datediff('day', 数据日期, '2023-09-05') <= 3) and 数据日期 = '2023-10-10' " + + "group by 部门 order by 总访问次数 desc limit 10"; replaceSql = SqlReplaceHelper.replaceAlias(sql); System.out.println(replaceSql); Assert.assertEquals( @@ -493,22 +563,23 @@ class SqlReplaceHelperTest { + "GROUP BY 部门 ORDER BY sum(访问次数) DESC LIMIT 10", replaceSql); - sql = "select 部门, sum(访问次数) as 访问次数 from 超音数 where " - + "(datediff('day', 数据日期, '2023-09-05') <= 3) and 数据日期 = '2023-10-10' " - + "group by 部门 order by 访问次数 desc limit 10"; + sql = + "select 部门, sum(访问次数) as 访问次数 from 超音数 where " + + "(datediff('day', 数据日期, '2023-09-05') <= 3) and 数据日期 = '2023-10-10' " + + "group by 部门 order by 访问次数 desc limit 10"; replaceSql = SqlReplaceHelper.replaceAlias(sql); System.out.println(replaceSql); Assert.assertEquals( "SELECT 部门, sum(访问次数) AS 访问次数 FROM 超音数 WHERE (datediff('day', 数据日期, " + "'2023-09-05') <= 3) AND 数据日期 = '2023-10-10' GROUP BY 部门 ORDER BY 访问次数 DESC LIMIT 10", replaceSql); - } @Test void testReplaceAggAliasOrderItem() { - String sql = "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 " - + "GROUP BY 部门 ORDER BY SUM(访问次数) DESC LIMIT 10) AS top10"; + String sql = + "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 " + + "GROUP BY 部门 ORDER BY SUM(访问次数) DESC LIMIT 10) AS top10"; String replaceSql = SqlReplaceHelper.replaceAggAliasOrderItem(sql); Assert.assertEquals( "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 " diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelperTest.java index 8116e0dd1..07a891afd 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectFunctionHelperTest.java @@ -4,41 +4,45 @@ import net.sf.jsqlparser.JSQLParserException; import org.junit.Assert; import org.junit.jupiter.api.Test; -/** - * SqlParserSelectHelper Test - */ +/** SqlParserSelectHelper Test */ class SqlSelectFunctionHelperTest { @Test void testHasAggregateFunction() throws JSQLParserException { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; boolean hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); - sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + sql = + "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); - sql = "SELECT count(1) FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice'" - + " AND publish_date = '11' ORDER BY pv DESC LIMIT 1"; + sql = + "SELECT count(1) FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice'" + + " AND publish_date = '11' ORDER BY pv DESC LIMIT 1"; hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); - sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " - + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"; + sql = + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"; hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, false); - sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" - + " AND user_id = 'alice' AND publish_date = '11'"; + sql = + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" + + " AND user_id = 'alice' AND publish_date = '11'"; hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, false); - sql = "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " - + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; + sql = + "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " + + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); } @@ -46,28 +50,33 @@ class SqlSelectFunctionHelperTest { @Test void testHasFunction() throws JSQLParserException { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; boolean hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "sum"); Assert.assertEquals(hasFunction, true); - sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + sql = + "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "count"); Assert.assertEquals(hasFunction, true); - sql = "select 部门,count (*) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + sql = + "select 部门,count (*) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "count"); Assert.assertEquals(hasFunction, true); - sql = "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " - + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; + sql = + "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " + + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "sum"); Assert.assertEquals(hasFunction, false); - sql = "select 部门,min (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + sql = + "select 部门,min (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "min"); Assert.assertEquals(hasFunction, true); @@ -75,12 +84,12 @@ class SqlSelectFunctionHelperTest { @Test void testHasAsterisk() { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; Assert.assertEquals(SqlSelectFunctionHelper.hasAsterisk(sql), false); - sql = "select * from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11'"; + sql = "select * from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11'"; Assert.assertEquals(SqlSelectFunctionHelper.hasAsterisk(sql), true); } } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelperTest.java index fce07dca1..ebf8fe1a7 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelperTest.java @@ -7,180 +7,205 @@ import org.junit.jupiter.api.Test; import java.util.List; -/** - * SqlParserSelectHelper Test - */ +/** SqlParserSelectHelper Test */ class SqlSelectHelperTest { @Test void testGetWhereFilterExpression() { - Select selectStatement = SqlSelectHelper.getSelect( - "select 用户名, 访问次数 from 超音数 where 用户名 in ('alice', 'lucy')"); + Select selectStatement = + SqlSelectHelper.getSelect( + "select 用户名, 访问次数 from 超音数 where 用户名 in ('alice', 'lucy')"); System.out.println(selectStatement); - List fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE " - + "sys_imp_date = '2023-08-08' AND YEAR(publish_date) = 2023 " - + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); + List fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE " + + "sys_imp_date = '2023-08-08' AND YEAR(publish_date) = 2023 " + + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " - + " AND YEAR(publish_date) = 2023 " - + " AND MONTH(publish_date) = 8" - + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + + " AND YEAR(publish_date) = 2023 " + + " AND MONTH(publish_date) = 8" + + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" - + " AND YEAR(publish_date) = 2023 " - + " AND MONTH(publish_date) = 8 AND DAY(publish_date) =20 " - + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" + + " AND YEAR(publish_date) = 2023 " + + " AND MONTH(publish_date) = 8 AND DAY(publish_date) =20 " + + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " - + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " - + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " - + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE " - + "user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE " + + "user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE " - + "user_id = 'alice' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE " + + "user_id = 'alice' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, user_id, field_a FROM s2 WHERE " - + "user_id like '%alice%' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE " + + "user_id like '%alice%' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, pv FROM s2 WHERE " - + "user_id like '%alice%' AND publish_date > 10000 " - + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, pv FROM s2 WHERE " + + "user_id like '%alice%' AND publish_date > 10000 " + + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, pv FROM s2 WHERE " - + "(user_id like '%alice%' AND publish_date > 10000) and sys_imp_date = '2023-08-08' " - + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, pv FROM s2 WHERE " + + "(user_id like '%alice%' AND publish_date > 10000) and sys_imp_date = '2023-08-08' " + + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, pv FROM s2 WHERE " - + "(user_id like '%alice%' AND publish_date > 10000) and song_name in " - + "('七里香','晴天') and sys_imp_date = '2023-08-08' " - + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, pv FROM s2 WHERE " + + "(user_id like '%alice%' AND publish_date > 10000) and song_name in " + + "('七里香','晴天') and sys_imp_date = '2023-08-08' " + + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, pv FROM s2 WHERE " - + "(user_id like '%alice%' AND publish_date > 10000) and song_name in (1,2) " - + "and sys_imp_date = '2023-08-08' " - + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, pv FROM s2 WHERE " + + "(user_id like '%alice%' AND publish_date > 10000) and song_name in (1,2) " + + "and sys_imp_date = '2023-08-08' " + + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "SELECT department, pv FROM s2 WHERE " - + "(user_id like '%alice%' AND publish_date > 10000) and 1 in (1) " - + "and sys_imp_date = '2023-08-08' " - + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT department, pv FROM s2 WHERE " + + "(user_id like '%alice%' AND publish_date > 10000) and 1 in (1) " + + "and sys_imp_date = '2023-08-08' " + + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression("SELECT sum(销量) / (SELECT sum(销量) FROM 营销月模型 " - + "WHERE MONTH(数据日期) = 9) FROM 营销月模型 WHERE 国家中文名 = '肯尼亚' AND MONTH(数据日期) = 9"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "SELECT sum(销量) / (SELECT sum(销量) FROM 营销月模型 " + + "WHERE MONTH(数据日期) = 9) FROM 营销月模型 WHERE 国家中文名 = '肯尼亚' AND MONTH(数据日期) = 9"); System.out.println(fieldExpression); - fieldExpression = SqlSelectHelper.getFilterExpression( - "select 等级, count(*) from 歌手 where 别名 = '港台' or 活跃区域 = '港台' and" - + " datediff('day', 数据日期, '2023-12-24') <= 0 group by 等级"); + fieldExpression = + SqlSelectHelper.getFilterExpression( + "select 等级, count(*) from 歌手 where 别名 = '港台' or 活跃区域 = '港台' and" + + " datediff('day', 数据日期, '2023-12-24') <= 0 group by 等级"); System.out.println(fieldExpression); - } @Test void testGetAllFields() { - List allFields = SqlSelectHelper.getAllSelectFields( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" - + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); + List allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" + + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); Assert.assertEquals(allFields.size(), 6); - allFields = SqlSelectHelper.getAllSelectFields( - "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date >= '2023-08-08'" - + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date >= '2023-08-08'" + + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); Assert.assertEquals(allFields.size(), 6); - allFields = SqlSelectHelper.getAllSelectFields( - "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice'" - + " and 发布日期 ='11' group by 部门 limit 1"); + allFields = + SqlSelectHelper.getAllSelectFields( + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice'" + + " and 发布日期 ='11' group by 部门 limit 1"); Assert.assertEquals(allFields.size(), 5); - allFields = SqlSelectHelper.getAllSelectFields( - "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "); Assert.assertEquals(allFields.size(), 3); - allFields = SqlSelectHelper.getAllSelectFields( - "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"); Assert.assertEquals(allFields.size(), 3); - allFields = SqlSelectHelper.getAllSelectFields( - "SELECT department, user_id, field_a FROM s2 WHERE " - + "(user_id = 'alice' AND publish_date = '11') and sys_imp_date " - + "= '2023-08-08' ORDER BY pv DESC LIMIT 1"); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT department, user_id, field_a FROM s2 WHERE " + + "(user_id = 'alice' AND publish_date = '11') and sys_imp_date " + + "= '2023-08-08' ORDER BY pv DESC LIMIT 1"); Assert.assertEquals(allFields.size(), 6); - allFields = SqlSelectHelper.getAllSelectFields( - "SELECT * FROM CSpider WHERE (评分 < (SELECT min(评分) FROM CSpider WHERE 语种 = '英文' ))" - + " AND 数据日期 = '2023-10-12'"); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT * FROM CSpider WHERE (评分 < (SELECT min(评分) FROM CSpider WHERE 语种 = '英文' ))" + + " AND 数据日期 = '2023-10-12'"); Assert.assertEquals(allFields.size(), 3); - allFields = SqlSelectHelper.getAllSelectFields("SELECT sum(销量) / (SELECT sum(销量) FROM 营销 " - + "WHERE MONTH(数据日期) = 9) FROM 营销 WHERE 国家中文名 = '中国' AND MONTH(数据日期) = 9"); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT sum(销量) / (SELECT sum(销量) FROM 营销 " + + "WHERE MONTH(数据日期) = 9) FROM 营销 WHERE 国家中文名 = '中国' AND MONTH(数据日期) = 9"); Assert.assertEquals(allFields.size(), 3); - allFields = SqlSelectHelper.getAllSelectFields( - "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC"); + allFields = + SqlSelectHelper.getAllSelectFields( + "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC"); Assert.assertEquals(allFields.size(), 2); } @@ -188,8 +213,9 @@ class SqlSelectHelperTest { @Test void testGetSelectFields() { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " - + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; List selectFields = SqlSelectHelper.getSelectFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); @@ -199,25 +225,28 @@ class SqlSelectHelperTest { @Test void testGetWhereFields() { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" - + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" + + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; List selectFields = SqlSelectHelper.getWhereFields(sql); Assert.assertEquals(selectFields.contains("发布日期"), true); Assert.assertEquals(selectFields.contains("数据日期"), true); Assert.assertEquals(selectFields.contains("用户"), true); - sql = "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'" - + " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1"; + sql = + "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'" + + " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1"; selectFields = SqlSelectHelper.getWhereFields(sql); Assert.assertEquals(selectFields.contains("发布日期"), true); Assert.assertEquals(selectFields.contains("数据日期"), true); Assert.assertEquals(selectFields.contains("用户"), true); - sql = "select 部门,用户 from 超音数 where" - + " (用户 = 'alice' and 发布日期 ='11') and 数据日期 = '2023-08-08' " - + "order by 访问次数 limit 1"; + sql = + "select 部门,用户 from 超音数 where" + + " (用户 = 'alice' and 发布日期 ='11') and 数据日期 = '2023-08-08' " + + "order by 访问次数 limit 1"; selectFields = SqlSelectHelper.getWhereFields(sql); Assert.assertEquals(selectFields.contains("发布日期"), true); @@ -228,14 +257,16 @@ class SqlSelectHelperTest { @Test void testGetOrderByFields() { - String sql = "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'" - + " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1"; + String sql = + "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'" + + " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1"; List selectFields = SqlSelectHelper.getOrderByFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); - sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "; + sql = + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "; selectFields = SqlSelectHelper.getOrderByFields(sql); Assert.assertEquals(selectFields.contains("pv"), true); @@ -244,40 +275,41 @@ class SqlSelectHelperTest { @Test void testGetGroupByFields() { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" - + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" + + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; List selectFields = SqlSelectHelper.getGroupByFields(sql); Assert.assertEquals(selectFields.contains("部门"), true); - } @Test void testGetHavingExpression() { - String sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " - + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; + String sql = + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; List leftExpressionList = SqlSelectHelper.getHavingExpression(sql); Assert.assertEquals(leftExpressionList.get(0).toString(), "sum(pv)"); - } @Test void testGetAggregateFields() { - String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" - + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" + + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; List selectFields = SqlSelectHelper.getAggregateFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); - } @Test void testGetTableName() { - String sql = "select 部门,sum (访问次数) from `超音数` where 数据日期 = '2023-08-08'" - + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; + String sql = + "select 部门,sum (访问次数) from `超音数` where 数据日期 = '2023-08-08'" + + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; String tableName = SqlSelectHelper.getTableName(sql); Assert.assertEquals(tableName, "超音数"); } @@ -285,20 +317,20 @@ class SqlSelectHelperTest { @Test void testGetPureSelectFields() { - String sql = "select TIMESTAMPDIFF(MONTH, 发布日期, '2018-06-01') from `超音数` " - + "where 数据日期 = '2023-08-08' and 用户 = 'alice'"; + String sql = + "select TIMESTAMPDIFF(MONTH, 发布日期, '2018-06-01') from `超音数` " + + "where 数据日期 = '2023-08-08' and 用户 = 'alice'"; List selectFields = SqlSelectHelper.gePureSelectFields(sql); Assert.assertEquals(selectFields.size(), 0); - sql = "select 发布日期,数据日期 from `超音数` where " - + "数据日期 = '2023-08-08' and 用户 = 'alice'"; + sql = "select 发布日期,数据日期 from `超音数` where " + "数据日期 = '2023-08-08' and 用户 = 'alice'"; selectFields = SqlSelectHelper.gePureSelectFields(sql); Assert.assertEquals(selectFields.size(), 2); - sql = "select 发布日期,数据日期,TIMESTAMPDIFF(MONTH, 发布日期, '2018-06-01') from `超音数` where " - + "数据日期 = '2023-08-08' and 用户 = 'alice'"; + sql = + "select 发布日期,数据日期,TIMESTAMPDIFF(MONTH, 发布日期, '2018-06-01') from `超音数` where " + + "数据日期 = '2023-08-08' and 用户 = 'alice'"; selectFields = SqlSelectHelper.gePureSelectFields(sql); Assert.assertEquals(selectFields.size(), 2); } - } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelperTest.java index 1a7dbd782..b24eb37b3 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelperTest.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.common.jsqlparser; - import org.junit.Assert; import org.junit.jupiter.api.Test; @@ -16,15 +15,19 @@ class SqlValidHelperTest { sql2 = "SELECT d,c,b,a FROM table1 WHERE column2 = 2 AND column1 = 1 order by a"; Assert.assertEquals(SqlValidHelper.equals(sql1, sql2), true); - sql1 = "SELECT a,sum(b),sum(c),sum(d) FROM table1 WHERE column1 = 1 AND column2 = 2 group by a order by a"; + sql1 = + "SELECT a,sum(b),sum(c),sum(d) FROM table1 WHERE column1 = 1 AND column2 = 2 group by a order by a"; - sql2 = "SELECT sum(d),sum(c),sum(b),a FROM table1 WHERE column2 = 2 AND column1 = 1 group by a order by a"; + sql2 = + "SELECT sum(d),sum(c),sum(b),a FROM table1 WHERE column2 = 2 AND column1 = 1 group by a order by a"; Assert.assertEquals(SqlValidHelper.equals(sql1, sql2), true); - sql1 = "SELECT a,sum(b),sum(c),sum(d) FROM table1 WHERE column1 = 1 AND column2 = 2 group by a order by a"; + sql1 = + "SELECT a,sum(b),sum(c),sum(d) FROM table1 WHERE column1 = 1 AND column2 = 2 group by a order by a"; - sql2 = "SELECT sum(d),sum(c),sum(b),a FROM table1 WHERE column2 = 2 AND column1 = 1 group by a order by a"; + sql2 = + "SELECT sum(d),sum(c),sum(b),a FROM table1 WHERE column2 = 2 AND column1 = 1 group by a order by a"; Assert.assertEquals(SqlValidHelper.equals(sql1, sql2), true); @@ -32,33 +35,35 @@ class SqlValidHelperTest { sql2 = "SELECT d,c,b,f FROM table1 WHERE column2 = 2 AND column1 = 1 order by a"; Assert.assertEquals(SqlValidHelper.equals(sql1, sql2), false); - sql1 = "SELECT\n" - + "页面,\n" - + "SUM(访问次数)\n" - + "FROM\n" - + "超音数\n" - + "WHERE\n" - + "数据日期 >= '2023-10-26'\n" - + "AND 数据日期 <= '2023-11-09'\n" - + "AND department = \"HR\"\n" - + "GROUP BY\n" - + "页面\n" - + "LIMIT\n" - + "365"; + sql1 = + "SELECT\n" + + "页面,\n" + + "SUM(访问次数)\n" + + "FROM\n" + + "超音数\n" + + "WHERE\n" + + "数据日期 >= '2023-10-26'\n" + + "AND 数据日期 <= '2023-11-09'\n" + + "AND department = \"HR\"\n" + + "GROUP BY\n" + + "页面\n" + + "LIMIT\n" + + "365"; - sql2 = "SELECT\n" - + "页面,\n" - + "SUM(访问次数)\n" - + "FROM\n" - + "超音数\n" - + "WHERE\n" - + "department = \"HR\"\n" - + "AND 数据日期 >= '2023-10-26'\n" - + "AND 数据日期 <= '2023-11-09'\n" - + "GROUP BY\n" - + "页面\n" - + "LIMIT\n" - + "365"; + sql2 = + "SELECT\n" + + "页面,\n" + + "SUM(访问次数)\n" + + "FROM\n" + + "超音数\n" + + "WHERE\n" + + "department = \"HR\"\n" + + "AND 数据日期 >= '2023-10-26'\n" + + "AND 数据日期 <= '2023-11-09'\n" + + "GROUP BY\n" + + "页面\n" + + "LIMIT\n" + + "365"; Assert.assertEquals(SqlValidHelper.equals(sql1, sql2), true); } @@ -82,4 +87,4 @@ class SqlValidHelperTest { Assert.assertEquals(SqlValidHelper.isValidSQL(sql1), false); } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ActionInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ActionInfo.java index f625e2082..7428d669e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ActionInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ActionInfo.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.api.pojo; import lombok.Data; + import java.util.List; @Data public class ActionInfo { private List out; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java index 218b24f01..4c9294bc7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java @@ -9,5 +9,4 @@ import java.util.List; public class AppConfig { private List items = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Cache.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Cache.java index cbc01f824..3f2eaee9d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Cache.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Cache.java @@ -10,9 +10,8 @@ public class Cache { @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"cache\":") - .append(cache); + sb.append("\"cache\":").append(cache); sb.append('}'); return sb.toString(); } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java index 8e6d22efc..c9be51461 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -15,5 +14,4 @@ public class DBColumn { private String dataType; private String comment; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java index 856e5664c..035c755e7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.api.pojo; import lombok.Data; + import java.util.List; @Data public class DataSetDetail { private List dataSetModelConfigs; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java index 2677e7bb0..5f2d79b26 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java @@ -65,7 +65,9 @@ public class DataSetSchema { allElements.addAll(getDimensions()); allElements.addAll(getMetrics()); return allElements.stream() - .collect(Collectors.toMap(SchemaElement::getBizName, SchemaElement::getName, (k1, k2) -> k1)); + .collect( + Collectors.toMap( + SchemaElement::getBizName, SchemaElement::getName, (k1, k2) -> k1)); } public TimeDefaultConfig getTagTypeTimeDefaultConfig() { @@ -97,28 +99,36 @@ public class DataSetSchema { public List getTagDefaultDimensions() { TagTypeDefaultConfig tagTypeDefaultConfig = getTagTypeDefaultConfig(); - if (Objects.isNull(tagTypeDefaultConfig) || Objects.isNull(tagTypeDefaultConfig.getDefaultDisplayInfo())) { + if (Objects.isNull(tagTypeDefaultConfig) + || Objects.isNull(tagTypeDefaultConfig.getDefaultDisplayInfo())) { return new ArrayList<>(); } - if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { - return tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds() - .stream().map(id -> { - SchemaElement metric = getElement(SchemaElementType.METRIC, id); - return metric; - }).filter(Objects::nonNull).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty( + tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { + return tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds().stream() + .map( + id -> { + SchemaElement metric = getElement(SchemaElementType.METRIC, id); + return metric; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } return new ArrayList<>(); } public List getTagDefaultMetrics() { TagTypeDefaultConfig tagTypeDefaultConfig = getTagTypeDefaultConfig(); - if (Objects.isNull(tagTypeDefaultConfig) || Objects.isNull(tagTypeDefaultConfig.getDefaultDisplayInfo())) { + if (Objects.isNull(tagTypeDefaultConfig) + || Objects.isNull(tagTypeDefaultConfig.getDefaultDisplayInfo())) { return new ArrayList<>(); } - if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { + if (CollectionUtils.isNotEmpty( + tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { return tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() .map(id -> getElement(SchemaElementType.DIMENSION, id)) - .filter(Objects::nonNull).collect(Collectors.toList()); + .filter(Objects::nonNull) + .collect(Collectors.toList()); } return new ArrayList<>(); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java index 38b9fad39..b54aec90d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo; - import lombok.Data; import java.util.ArrayList; @@ -9,9 +8,8 @@ import java.util.List; @Data public class DefaultDisplayInfo { - //When displaying tag selection results, the information displayed by default + // When displaying tag selection results, the information displayed by default private List dimensionIds = new ArrayList<>(); private List metricIds = new ArrayList<>(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java index 4bd8e4e39..16d1a6e8d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java @@ -5,7 +5,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - @Data @AllArgsConstructor @NoArgsConstructor @@ -44,8 +43,14 @@ public class Dim { this.isTag = isTag; } - public Dim(String name, String type, String expr, String dateFormat, DimensionTimeTypeParams typeParams, - Integer isCreateDimension, String bizName) { + public Dim( + String name, + String type, + String expr, + String dateFormat, + DimensionTimeTypeParams typeParams, + Integer isCreateDimension, + String bizName) { this.name = name; this.type = type; this.expr = expr; @@ -56,15 +61,17 @@ public class Dim { } public static Dim getDefault() { - return new Dim("日期", "time", "2023-05-28", + return new Dim( + "日期", + "time", + "2023-05-28", Constants.DAY_FORMAT, new DimensionTimeTypeParams("true", "day"), - 0, "imp_date" - ); + 0, + "imp_date"); } public String getFieldName() { return bizName; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java index a21240bce..3fc060fbe 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java @@ -8,18 +8,12 @@ import java.util.List; @Data public class DimValueMap { - /** - * dimension value in db - */ + /** dimension value in db */ private String techName; - /** - * dimension value for result show - */ + /** dimension value for result show */ private String bizName; - /** - * dimension value for user query - */ + /** dimension value for user query */ private List alias = new ArrayList<>(); -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimensionTimeTypeParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimensionTimeTypeParams.java index a29cb7a56..f3aa70c28 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimensionTimeTypeParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimensionTimeTypeParams.java @@ -4,7 +4,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - @Data @AllArgsConstructor @NoArgsConstructor @@ -13,5 +12,4 @@ public class DimensionTimeTypeParams { private String isPrimary = "true"; private String timeGranularity = "day"; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java index 1f39f9813..57aa44b8f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java @@ -23,5 +23,4 @@ public class DrillDownDimension { this.dimensionId = dimensionId; this.necessary = necessary; } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java index ac9c3e3b2..c5fc5b333 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java @@ -13,14 +13,9 @@ import java.util.List; @NoArgsConstructor public class Entity { - - /** - * uniquely identifies an entity - */ + /** uniquely identifies an entity */ private Long entityId; - /** - * entity name list - */ + /** entity name list */ private List names; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java index c9a3cbe26..306a7474c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java @@ -12,5 +12,4 @@ public class EntityInfo { private List dimensions = new ArrayList<>(); private List metrics = new ArrayList<>(); private String entityId; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java index abd225849..0416a694d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java @@ -14,5 +14,4 @@ public class Field { private String fieldName; private String dataType; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java index 4e796fb81..974143cff 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java @@ -10,5 +10,4 @@ import lombok.NoArgsConstructor; public class FieldParam { private String fieldName; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java index 81c7e645d..ed3667af4 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java @@ -13,9 +13,7 @@ public class Identify { private String name; - /** - * primary, foreign - */ + /** primary, foreign */ private String type; private String bizName; @@ -40,5 +38,4 @@ public class Identify { public String getFieldName() { return bizName; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Item.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Item.java index 9b9c4d824..4a8897d84 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Item.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Item.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.ApiItemType; import lombok.Data; + import java.util.List; @Data @@ -23,5 +24,4 @@ public class Item { public String getValue() { return name; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemDateFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemDateFilter.java index 642025f47..9cf0aa05a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemDateFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemDateFilter.java @@ -1,17 +1,17 @@ package com.tencent.supersonic.headless.api.pojo; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NonNull; import lombok.ToString; +import java.util.List; + @Data @AllArgsConstructor @ToString public class ItemDateFilter { private List itemIds; - @NonNull - private String type; -} \ No newline at end of file + @NonNull private String type; +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java index 54c48785b..244695893 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java @@ -6,10 +6,7 @@ import lombok.Data; import java.util.ArrayList; import java.util.List; -/** - * 导入字典的可选配置 - */ - +/** 导入字典的可选配置 */ @Data public class ItemValueConfig { @@ -19,4 +16,4 @@ public class ItemValueConfig { private List ruleList = new ArrayList<>(); private Long limit; private DateConf dateConf; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java index e71f23c60..b39521be3 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java @@ -38,5 +38,4 @@ public class Measure { public String getFieldName() { return expr; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MeasureParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MeasureParam.java index aaa0e276d..146573809 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MeasureParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MeasureParam.java @@ -19,5 +19,4 @@ public class MeasureParam { this.bizName = bizName; this.constraint = constraint; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java index e38121c40..c4e369468 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java @@ -6,7 +6,6 @@ import lombok.NoArgsConstructor; import java.util.List; - @Data @NoArgsConstructor public class MetaFilter { @@ -52,19 +51,34 @@ public class MetaFilter { return false; } MetaFilter that = (MetaFilter) o; - return Objects.equal(id, that.id) && Objects.equal(name, that.name) - && Objects.equal(bizName, that.bizName) && Objects.equal( - createdBy, that.createdBy) && Objects.equal(modelIds, that.modelIds) - && Objects.equal(domainId, that.domainId) && Objects.equal( - dataSetId, that.dataSetId) && Objects.equal(sensitiveLevel, that.sensitiveLevel) - && Objects.equal(status, that.status) && Objects.equal(key, - that.key) && Objects.equal(ids, that.ids) && Objects.equal( - fieldsDepend, that.fieldsDepend); + return Objects.equal(id, that.id) + && Objects.equal(name, that.name) + && Objects.equal(bizName, that.bizName) + && Objects.equal(createdBy, that.createdBy) + && Objects.equal(modelIds, that.modelIds) + && Objects.equal(domainId, that.domainId) + && Objects.equal(dataSetId, that.dataSetId) + && Objects.equal(sensitiveLevel, that.sensitiveLevel) + && Objects.equal(status, that.status) + && Objects.equal(key, that.key) + && Objects.equal(ids, that.ids) + && Objects.equal(fieldsDepend, that.fieldsDepend); } @Override public int hashCode() { - return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId, - dataSetId, sensitiveLevel, status, key, ids, fieldsDepend); + return Objects.hashCode( + id, + name, + bizName, + createdBy, + modelIds, + domainId, + dataSetId, + sensitiveLevel, + status, + key, + ids, + fieldsDepend); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByFieldParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByFieldParams.java index 002c4fc10..8006707c3 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByFieldParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByFieldParams.java @@ -9,5 +9,4 @@ import java.util.List; public class MetricDefineByFieldParams extends MetricDefineParams { private List fields = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java index a2df4d756..c186d8b67 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java @@ -2,11 +2,11 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.collect.Lists; import lombok.Data; + import java.util.List; @Data public class MetricDefineByMeasureParams extends MetricDefineParams { private List measures = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMetricParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMetricParams.java index d0058033b..eb8c5529f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMetricParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMetricParams.java @@ -9,5 +9,4 @@ import java.util.List; public class MetricDefineByMetricParams extends MetricDefineParams { private List metrics = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java index 47783f5dc..32ad572bf 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java @@ -8,5 +8,4 @@ public abstract class MetricDefineParams { private String expr; private String filterSql; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java index 299b710fc..4a674ef9c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java @@ -12,5 +12,4 @@ public class MetricInfo { private String value; private String date; private Map statistics; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java index 8d635cf48..aaf067c82 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java @@ -12,5 +12,4 @@ public class MetricParam { private Long id; private String bizName; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricQueryDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricQueryDefaultConfig.java index 5e4ce8abf..328965a48 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricQueryDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricQueryDefaultConfig.java @@ -12,7 +12,6 @@ public class MetricQueryDefaultConfig extends RecordInfo { private String userName; - //string of queryStruct + // string of queryStruct private String defaultConfig; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java index c4b26e6d1..9740891de 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java @@ -14,5 +14,4 @@ public class MetricTable { private List dimensions = Lists.newArrayList(); private String where; private AggOption aggOption = AggOption.DEFAULT; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java index 0d6dc4c90..e5eac0cff 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java @@ -9,5 +9,4 @@ public class MetricTypeDefaultConfig { private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(7, Constants.DAY, TimeMode.RECENT); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index 8ddd0f076..aa46a1b54 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -53,23 +53,33 @@ public class ModelDetail { return fields; } List fieldList = Lists.newArrayList(); - //Compatible with older versions + // Compatible with older versions if (!CollectionUtils.isEmpty(identifiers)) { - fieldList.addAll(identifiers.stream() - .map(identify -> Field.builder().fieldName(identify.getFieldName()).build()) - .collect(Collectors.toSet())); + fieldList.addAll( + identifiers.stream() + .map( + identify -> + Field.builder() + .fieldName(identify.getFieldName()) + .build()) + .collect(Collectors.toSet())); } if (!CollectionUtils.isEmpty(dimensions)) { - fieldList.addAll(dimensions.stream() - .map(dim -> Field.builder().fieldName(dim.getFieldName()).build()) - .collect(Collectors.toSet())); + fieldList.addAll( + dimensions.stream() + .map(dim -> Field.builder().fieldName(dim.getFieldName()).build()) + .collect(Collectors.toSet())); } if (!CollectionUtils.isEmpty(measures)) { - fieldList.addAll(measures.stream() - .map(measure -> Field.builder().fieldName(measure.getFieldName()).build()) - .collect(Collectors.toSet())); + fieldList.addAll( + measures.stream() + .map( + measure -> + Field.builder() + .fieldName(measure.getFieldName()) + .build()) + .collect(Collectors.toSet())); } return fieldList; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java index dbf36dce5..1068e6654 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.api.pojo; -import lombok.Data; - import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; +import lombok.Data; + @Data public class Param { @@ -14,12 +14,10 @@ public class Param { @NotNull(message = "Invalid parameter value") private String value; - public Param() { - } + public Param() {} public Param(String name, String value) { this.name = name; this.value = value; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java index 1f077e2ea..e1ba567aa 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java @@ -8,6 +8,4 @@ public class QueryConfig { private TagTypeDefaultConfig tagTypeDefaultConfig = new TagTypeDefaultConfig(); private MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig(); - - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryDataType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryDataType.java index 08dec1bdf..7f92abb19 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryDataType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryDataType.java @@ -5,4 +5,4 @@ public enum QueryDataType { DIMENSION, TAG, ALL -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java index 6b133974d..401d18204 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; import lombok.Data; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -36,5 +37,4 @@ public class QueryParam { private String where; private List order; private boolean nativeQuery = false; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryResult.java index 54cc6af76..70597f9b7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryResult.java @@ -15,4 +15,4 @@ public class QueryResult implements Serializable { private int pageSize = -1; private long totalCount = -1; private List resultList = new ArrayList(); -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java index 6123b5ef8..5772222da 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java @@ -19,14 +19,11 @@ public class QueryStat { private Long dataSetId; private String user; private String createdAt; - /** - * corresponding type, such as sql, struct, etc - */ + /** corresponding type, such as sql, struct, etc */ private String queryType; - /** - * NORMAL, PRE_FLUSH - */ + /** NORMAL, PRE_FLUSH */ private Integer queryTypeBack; + private String querySqlCmd; private String querySqlCmdMd5; private String queryStructCmd; @@ -228,5 +225,4 @@ public class QueryStat { public List getDimensionListBizName() { return JSONObject.parseArray(dimensions, String.class); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelateDimension.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelateDimension.java index 4ecb1c468..9f3d69877 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelateDimension.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelateDimension.java @@ -5,6 +5,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; + import java.util.List; @Data @@ -14,5 +15,4 @@ import java.util.List; public class RelateDimension { List drillDownDimensions = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java index e8fd6d014..77bbd12eb 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -15,5 +14,4 @@ public class RelatedSchemaElement { private Long dimensionId; private boolean isNecessary; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RuleInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RuleInfo.java index 318cb2513..6a8b48a50 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RuleInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RuleInfo.java @@ -11,11 +11,9 @@ public class RuleInfo { private List parameters; public enum Mode { - /** - * BEFORE, some days ago - * RECENT, the last few days - * EXIST, there was some information - */ - BEFORE, RECENT, EXIST + /** BEFORE, some days ago RECENT, the last few days EXIST, there was some information */ + BEFORE, + RECENT, + EXIST } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java index 2707f6062..fbe001269 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java @@ -40,8 +40,7 @@ public class SchemaElement implements Serializable { private int isTag; private String description; private boolean descriptionMapped; - @Builder.Default - private Map extInfo = new HashMap<>(); + @Builder.Default private Map extInfo = new HashMap<>(); @Override public boolean equals(Object o) { @@ -52,8 +51,9 @@ public class SchemaElement implements Serializable { return false; } SchemaElement schemaElement = (SchemaElement) o; - return Objects.equal(dataSetId, schemaElement.dataSetId) && Objects.equal(id, - schemaElement.id) && Objects.equal(name, schemaElement.name) + return Objects.equal(dataSetId, schemaElement.dataSetId) + && Objects.equal(id, schemaElement.id) + && Objects.equal(name, schemaElement.name) && Objects.equal(bizName, schemaElement.bizName) && Objects.equal(type, schemaElement.type); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java index 1ec8511ed..2509961e8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java @@ -23,5 +23,4 @@ public class SchemaElementMatch { public boolean isFullMatched() { return 1.0 == similarity; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java index 29aa91e50..46bb86b96 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java @@ -40,7 +40,8 @@ public class SchemaItem extends RecordInfo { return false; } SchemaItem that = (SchemaItem) o; - return Objects.equal(id, that.id) && Objects.equal(name, that.name) + return Objects.equal(id, that.id) + && Objects.equal(name, that.name) && Objects.equal(bizName, that.bizName) && typeEnum == that.typeEnum; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java index 06db1f387..d7c936845 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java @@ -48,5 +48,4 @@ public class SchemaMapInfo { public boolean needContinueMap() { return CollectionUtils.isNotEmpty(getTermDescriptionToMap()); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java index bea54fac6..98739e715 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java @@ -1,24 +1,19 @@ package com.tencent.supersonic.headless.api.pojo; +import lombok.Data; + import java.util.ArrayList; import java.util.List; -import lombok.Data; @Data public class SchemaValueMap { - /** - * dimension value in db - */ + /** dimension value in db */ private String techName; - /** - * dimension value for result show - */ + /** dimension value for result show */ private String bizName; - /** - * dimension value for user query - */ + /** dimension value for user query */ private List alias = new ArrayList<>(); -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index b1fc009b1..db0edb029 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -68,5 +68,4 @@ public class SemanticParseInfo { } return dataSet.getDataSetId(); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java index fdbe73495..610318a41 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java @@ -59,7 +59,11 @@ public class SemanticSchema implements Serializable { public Map getDataSetIdToName() { return dataSetSchemaList.stream() - .collect(Collectors.toMap(a -> a.getDataSet().getId(), a -> a.getDataSet().getName(), (k1, k2) -> k1)); + .collect( + Collectors.toMap( + a -> a.getDataSet().getId(), + a -> a.getDataSet().getName(), + (k1, k2) -> k1)); } public List getDimensionValues() { @@ -115,8 +119,10 @@ public class SemanticSchema implements Serializable { public List getTags(Long dataSetId) { List tags = new ArrayList<>(); - dataSetSchemaList.stream().filter(schemaElement -> - dataSetId.equals(schemaElement.getDataSet().getDataSetId())) + dataSetSchemaList.stream() + .filter( + schemaElement -> + dataSetId.equals(schemaElement.getDataSet().getDataSetId())) .forEach(d -> tags.addAll(d.getTags())); return tags; } @@ -127,7 +133,8 @@ public class SemanticSchema implements Serializable { return terms; } - private List getElementsByDataSetId(Long dataSetId, List elements) { + private List getElementsByDataSetId( + Long dataSetId, List elements) { return elements.stream() .filter(schemaElement -> dataSetId.equals(schemaElement.getDataSetId())) .collect(Collectors.toList()); @@ -154,7 +161,10 @@ public class SemanticSchema implements Serializable { if (CollectionUtils.isEmpty(dataSetSchemaList)) { return new HashMap<>(); } - return dataSetSchemaList.stream().collect(Collectors.toMap(dataSetSchema - -> dataSetSchema.getDataSet().getDataSetId(), dataSetSchema -> dataSetSchema)); + return dataSetSchemaList.stream() + .collect( + Collectors.toMap( + dataSetSchema -> dataSetSchema.getDataSet().getDataSetId(), + dataSetSchema -> dataSetSchema)); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java index 8aabc8ae0..c7be7f45f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo; - import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import lombok.Data; @@ -10,5 +9,4 @@ public class SingleItemQueryResult { private Item item; private SemanticQueryResp result; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java index 596cfe873..c83b75901 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java @@ -13,5 +13,4 @@ public class SqlInfo { // SQL to be executed finally private String querySQL; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java index bdee9e5dd..cac26a21a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java @@ -12,5 +12,3 @@ public class SqlVariable { private VariableValueType valueType; private List defaultValues = Lists.newArrayList(); } - - diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java index f9e72821c..fddc79f3a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java @@ -7,7 +7,6 @@ public class TagTypeDefaultConfig { private DefaultDisplayInfo defaultDisplayInfo; - //default time to filter tag selection results + // default time to filter tag selection results private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java index c1ae17aa6..6c803b646 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo; - import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeMode; import lombok.AllArgsConstructor; @@ -12,18 +11,11 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class TimeDefaultConfig { - - /** - * default time span unit - */ + /** default time span unit */ private Integer unit = 1; - /** - * default time type: day - * DAY, WEEK, MONTH, YEAR - */ + /** default time type: day DAY, WEEK, MONTH, YEAR */ private String period = Constants.DAY; private TimeMode timeMode = TimeMode.LAST; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java index 58026c85f..06ecd5322 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java @@ -16,4 +16,4 @@ public class ValueDistribution { private Long valueCount; private Double ratio; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java index 1c46d8d6b..72afcbb66 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java @@ -1,9 +1,8 @@ package com.tencent.supersonic.headless.api.pojo.enums; /** - * Aggregation type of metric when query metric without aggregation method - * NATIVE: will not use Aggregation - * DEFAULT: will use the aggregation method define in the model + * Aggregation type of metric when query metric without aggregation method NATIVE: will not use + * Aggregation DEFAULT: will use the aggregation method define in the model */ public enum AggOption { NATIVE, diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AppStatus.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AppStatus.java index 43ac44fac..4bf590529 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AppStatus.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AppStatus.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum AppStatus { - INIT(0), ONLINE(1), OFFLINE(2), diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java index 27b2ce4cf..c30969613 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java @@ -7,4 +7,4 @@ public enum ChatWorkflowState { TRANSLATING, PROCESSING, FINISHED -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java index 8ab8c561b..51779fc5a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java @@ -1,36 +1,69 @@ package com.tencent.supersonic.headless.api.pojo.enums; - import com.tencent.supersonic.common.pojo.Constants; + import java.util.HashSet; import java.util.Set; public enum DataType { - MYSQL("mysql", "mysql", "com.mysql.cj.jdbc.Driver", "`", "`", "'", "'"), HIVE2("hive2", "hive", "org.apache.hive.jdbc.HiveDriver", "`", "`", "`", "`"), ORACLE("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""), - SQLSERVER("sqlserver", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), + SQLSERVER( + "sqlserver", + "sqlserver", + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "\"", + "\"", + "\"", + "\""), H2("h2", "h2", "org.h2.Driver", "`", "`", "\"", "\""), - PHOENIX("phoenix", "hbase phoenix", "org.apache.phoenix.jdbc.PhoenixDriver", "", "", "\"", "\""), + PHOENIX( + "phoenix", + "hbase phoenix", + "org.apache.phoenix.jdbc.PhoenixDriver", + "", + "", + "\"", + "\""), MONGODB("mongo", "mongodb", "mongodb.jdbc.MongoDriver", "`", "`", "\"", "\""), - ELASTICSEARCH("elasticsearch", "elasticsearch", "com.amazon.opendistroforelasticsearch.jdbc.Driver", "", "", "'", + ELASTICSEARCH( + "elasticsearch", + "elasticsearch", + "com.amazon.opendistroforelasticsearch.jdbc.Driver", + "", + "", + "'", "'"), PRESTO("presto", "presto", "com.facebook.presto.jdbc.PrestoDriver", "\"", "\"", "\"", "\""), MOONBOX("moonbox", "moonbox", "moonbox.jdbc.MbDriver", "`", "`", "`", "`"), - CASSANDRA("cassandra", "cassandra", "com.github.adejanovski.cassandra.jdbc.CassandraDriver", "", "", "'", "'"), + CASSANDRA( + "cassandra", + "cassandra", + "com.github.adejanovski.cassandra.jdbc.CassandraDriver", + "", + "", + "'", + "'"), - CLICKHOUSE("clickhouse", "clickhouse", "ru.yandex.clickhouse.ClickHouseDriver", "", "", "\"", "\""), + CLICKHOUSE( + "clickhouse", + "clickhouse", + "ru.yandex.clickhouse.ClickHouseDriver", + "", + "", + "\"", + "\""), KYLIN("kylin", "kylin", "org.apache.kylin.jdbc.Driver", "\"", "\"", "\"", "\""), @@ -52,8 +85,14 @@ public enum DataType { private String aliasPrefix; private String aliasSuffix; - DataType(String feature, String desc, String driver, String keywordPrefix, String keywordSuffix, - String aliasPrefix, String aliasSuffix) { + DataType( + String feature, + String desc, + String driver, + String keywordPrefix, + String keywordSuffix, + String aliasPrefix, + String aliasSuffix) { this.feature = feature; this.desc = desc; this.driver = driver; @@ -66,7 +105,8 @@ public enum DataType { public static DataType urlOf(String jdbcUrl) throws RuntimeException { String url = jdbcUrl.toLowerCase().trim(); for (DataType dataTypeEnum : values()) { - if (url.startsWith(String.format(Constants.JDBC_PREFIX_FORMATTER, dataTypeEnum.feature))) { + if (url.startsWith( + String.format(Constants.JDBC_PREFIX_FORMATTER, dataTypeEnum.feature))) { return dataTypeEnum; } } @@ -108,4 +148,4 @@ public enum DataType { public String getAliasSuffix() { return aliasSuffix; } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java index 76264f4cb..d1a7d5955 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum DimensionType { - categorical, time, partition_time, diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java index 30f56fb25..12992c8cc 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java @@ -1,8 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; - public enum EngineType { - TDW(0, "tdw"), MYSQL(1, "mysql"), DORIS(2, "doris"), @@ -12,7 +10,6 @@ public enum EngineType { POSTGRESQL(6, "postgresql"), OTHER(7, "other"); - private Integer code; private String name; @@ -38,5 +35,4 @@ public enum EngineType { } throw new IllegalArgumentException("Invalid value: " + value); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/IdentifyType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/IdentifyType.java index c0d7b324c..dbc65533b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/IdentifyType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/IdentifyType.java @@ -1,8 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum IdentifyType { - primary, foreign, - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricDefineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricDefineType.java index 1679caefc..611447d23 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricDefineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricDefineType.java @@ -1,9 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum MetricDefineType { - FIELD, MEASURE, METRIC - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java index 4007eb1fe..cc6e6c564 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java @@ -1,13 +1,12 @@ package com.tencent.supersonic.headless.api.pojo.enums; - import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; + import java.util.List; import java.util.Objects; public enum MetricType { - ATOMIC, DERIVED; @@ -25,7 +24,8 @@ public enum MetricType { return Objects.nonNull(metricType) && metricType.equals(DERIVED); } - public static Boolean isDerived(MetricDefineType metricDefineType, MetricDefineByMeasureParams typeParams) { + public static Boolean isDerived( + MetricDefineType metricDefineType, MetricDefineByMeasureParams typeParams) { if (MetricDefineType.METRIC.equals(metricDefineType)) { return true; } @@ -37,8 +37,8 @@ public enum MetricType { if (measures.size() > 1) { return true; } - if (measures.size() == 1 && measures.get(0).getBizName() - .equalsIgnoreCase(typeParams.getExpr())) { + if (measures.size() == 1 + && measures.get(0).getBizName().equalsIgnoreCase(typeParams.getExpr())) { return false; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java index e40c34b8e..d3a24d191 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java @@ -1,18 +1,15 @@ package com.tencent.supersonic.headless.api.pojo.enums; /** - * model datasource define type: - * sql_query : dataSet sql begin as select - * table_query: dbName.tableName + * model datasource define type: sql_query : dataSet sql begin as select table_query: + * dbName.tableName */ public enum ModelDefineType { - SQL_QUERY("sql_query"), TABLE_QUERY("table_query"); private String name; - ModelDefineType(String name) { this.name = name; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelSourceType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelSourceType.java index 085d29e9f..e85928e09 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelSourceType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelSourceType.java @@ -1,10 +1,9 @@ package com.tencent.supersonic.headless.api.pojo.enums; import java.util.Objects; + /** - * model source table type: - * FULL: table without time partition - * PARTITION: table with partition field + * model source table type: FULL: table without time partition PARTITION: table with partition field * ZIPPER: table with slowly changing dimension */ public enum ModelSourceType { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java index cc884cbb4..2446c4530 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.enums; - public enum QueryMethod { SQL("SQL"), @@ -24,5 +23,4 @@ public enum QueryMethod { public String getValue() { return value; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryOptMode.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryOptMode.java index b52ac8652..e0d902e50 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryOptMode.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryOptMode.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum QueryOptMode { - NONE, MATERIALIZATION diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryRuleType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryRuleType.java index 4a6db0c0a..3bde82b16 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryRuleType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryRuleType.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum QueryRuleType { - ADD_DATE, ADD_SELECT } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryTypeBack.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryTypeBack.java index c3aff9910..50308b9b1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryTypeBack.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryTypeBack.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.enums; - public enum QueryTypeBack { NORMAL("NORMAL", 0), @@ -30,5 +29,4 @@ public enum QueryTypeBack { public Integer getState() { return state; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java index 878cc1fa6..1f88057ea 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum SchemaType { - DATASET, MODEL } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SemanticType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SemanticType.java index 4f3289e6a..b49d147a9 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SemanticType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SemanticType.java @@ -1,10 +1,8 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum SemanticType { - CATEGORY, ID, DATE, NUMBER - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java index 0784ab27c..4257c9b36 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum TagDefineType { - FIELD, DIMENSION, METRIC, diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java index e5098baf7..c474056c2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java @@ -21,7 +21,8 @@ public enum TagType { } public static TagType getType(TagDefineType tagDefineType) { - return Objects.nonNull(tagDefineType) && TagDefineType.TAG.equals(tagDefineType) ? TagType.DERIVED + return Objects.nonNull(tagDefineType) && TagDefineType.TAG.equals(tagDefineType) + ? TagType.DERIVED : TagType.ATOMIC; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppQueryReq.java index 05407af24..854a47d50 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppQueryReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.request; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.PageBaseReq; import com.tencent.supersonic.headless.api.pojo.enums.AppStatus; @@ -25,5 +24,4 @@ public class AppQueryReq extends PageBaseReq { } return appStatus.stream().map(AppStatus::getCode).collect(Collectors.toList()); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppReq.java index f2005ae9f..58c9773be 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/AppReq.java @@ -31,5 +31,4 @@ public class AppReq extends RecordInfo { } return String.join(",", owners); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/BatchDownloadReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/BatchDownloadReq.java index bda5c7452..74f7c727c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/BatchDownloadReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/BatchDownloadReq.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.tencent.supersonic.common.pojo.DateConf; import lombok.Data; + import java.util.List; @Data @@ -12,5 +13,4 @@ public class BatchDownloadReq { private DateConf dateInfo; private boolean isTransform = true; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java index fe7711541..d4761e932 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java @@ -13,5 +13,4 @@ public class CanvasReq extends RecordInfo { private String type; private String config; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ClassReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ClassReq.java index 45bc6119a..97285e424 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ClassReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ClassReq.java @@ -12,5 +12,4 @@ public class ClassReq extends SchemaItem { private Long tagObjectId; private Long parentId; private List itemIds; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java index f0f965670..d7e6ba4be 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java @@ -29,5 +29,4 @@ public class DataSetFilterReq { public void addDataSet(Long dataSetId) { dataSetIds.add(dataSetId); } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java index f9d7c7dd4..e3653df27 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java @@ -35,5 +35,4 @@ public class DataSetReq extends SchemaItem { } return String.join(",", adminOrgs); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DatabaseReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DatabaseReq.java index 7f20265f6..17bb75309 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DatabaseReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DatabaseReq.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; -import java.util.List; +import java.util.List; @Data public class DatabaseReq extends RecordInfo { @@ -35,5 +35,4 @@ public class DatabaseReq extends RecordInfo { private List admins = Lists.newArrayList(); private List viewers = Lists.newArrayList(); - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DateInfoReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DateInfoReq.java index c1d1968cf..61122aea7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DateInfoReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DateInfoReq.java @@ -1,12 +1,13 @@ package com.tencent.supersonic.headless.api.pojo.request; -import java.util.ArrayList; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; +import java.util.ArrayList; +import java.util.List; + @Data @ToString @AllArgsConstructor @@ -21,7 +22,8 @@ public class DateInfoReq { private String datePeriod; private List unavailableDateList = new ArrayList<>(); - public DateInfoReq(String type, Long itemId, String dateFormat, String startDate, String endDate) { + public DateInfoReq( + String type, Long itemId, String dateFormat, String startDate, String endDate) { this.type = type; this.itemId = itemId; this.dateFormat = dateFormat; @@ -29,7 +31,12 @@ public class DateInfoReq { this.endDate = endDate; } - public DateInfoReq(String type, Long itemId, String dateFormat, String startDate, String endDate, + public DateInfoReq( + String type, + Long itemId, + String dateFormat, + String startDate, + String endDate, List unavailableDateList) { this.type = type; this.itemId = itemId; @@ -38,5 +45,4 @@ public class DateInfoReq { this.endDate = endDate; this.unavailableDateList = unavailableDateList; } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java index d2e9fd167..b827000a3 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java @@ -12,4 +12,4 @@ public class DictItemFilter { private TypeEnums type; private Long itemId; private StatusEnum status; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java index 22ad7cc5a..3427f9ab6 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java @@ -1,27 +1,20 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; import lombok.Data; -import javax.validation.constraints.NotNull; - @Data public class DictItemReq { private Long id; - @NotNull - private TypeEnums type; - @NotNull - private Long itemId; + @NotNull private TypeEnums type; + @NotNull private Long itemId; private ItemValueConfig config; - /** - * ONLINE - 正常更新 - * OFFLINE - 停止更新,但字典文件不删除 - * DELETED - 停止更新,且删除字典文件 - */ - @NotNull - private StatusEnum status; -} \ No newline at end of file + /** ONLINE - 正常更新 OFFLINE - 停止更新,但字典文件不删除 DELETED - 停止更新,且删除字典文件 */ + @NotNull private StatusEnum status; +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java index b4726af44..4b4a7a4c1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java @@ -1,16 +1,14 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.common.pojo.enums.TypeEnums; import lombok.Builder; import lombok.Data; -import javax.validation.constraints.NotNull; - @Data @Builder public class DictSingleTaskReq { - @NotNull - private TypeEnums type; - @NotNull - private Long itemId; -} \ No newline at end of file + @NotNull private TypeEnums type; + @NotNull private Long itemId; +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java index ec47dd586..dca5e5870 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java @@ -13,4 +13,4 @@ public class DictValueReq extends PageBaseReq { private Long itemId; private TypeEnums type = TypeEnums.DIMENSION; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java index 46078e7ee..ed6a7fc15 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import lombok.Data; -import javax.validation.constraints.NotNull; import java.util.List; import java.util.Map; @@ -19,7 +20,7 @@ public class DimensionReq extends SchemaItem { @NotNull(message = "expr can not be null") private String expr; - //DATE ID CATEGORY + // DATE ID CATEGORY private String semanticType = "CATEGORY"; private String alias; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java index 7f466b5c7..61d326956 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.common.pojo.DateConf; import lombok.Data; import org.apache.commons.lang3.StringUtils; -import javax.validation.constraints.NotNull; + import java.util.Set; @Data @@ -11,15 +13,13 @@ public class DimensionValueReq { private Integer agentId; - @NotNull - private Long elementID; + @NotNull private Long elementID; private Long modelId; private String bizName; - @NotNull - private String value; + @NotNull private String value; private Set dataSetIds; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainReq.java index 85c481d51..c67082058 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainReq.java @@ -1,13 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; - import com.tencent.supersonic.headless.api.pojo.SchemaItem; import lombok.Data; import java.util.ArrayList; import java.util.List; - @Data public class DomainReq extends SchemaItem { @@ -38,5 +36,4 @@ public class DomainReq extends SchemaItem { public String getAdminOrg() { return String.join(",", adminOrgs); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainUpdateReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainUpdateReq.java index 74669c661..25fd81bb3 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainUpdateReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DomainUpdateReq.java @@ -1,12 +1,9 @@ package com.tencent.supersonic.headless.api.pojo.request; - import lombok.Data; - @Data public class DomainUpdateReq extends DomainReq { private Long id; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DownloadMetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DownloadMetricReq.java index 93ae72097..c25a11dc5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DownloadMetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DownloadMetricReq.java @@ -10,5 +10,4 @@ public class DownloadMetricReq extends QueryMetricReq { public void setIsTransform(boolean isTransform) { this.isTransform = isTransform; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java index 6e7ba227f..fc6509297 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.request; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Builder; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/FieldRemovedReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/FieldRemovedReq.java index c539e2843..f27a7b29e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/FieldRemovedReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/FieldRemovedReq.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import lombok.Data; + import java.util.List; @Data @@ -9,5 +10,4 @@ public class FieldRemovedReq { private Long modelId; private List fields; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemUseReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemUseReq.java index 2ca79ea4e..7cf3994d1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemUseReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemUseReq.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; -import java.util.List; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; +import java.util.List; + @Data @ToString @NoArgsConstructor @@ -20,6 +21,7 @@ public class ItemUseReq { this.startTime = startTime; this.modelId = modelId; } + public ItemUseReq(String startTime, List modelIds) { this.startTime = startTime; this.modelIds = modelIds; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java index 3fe99a0aa..297e3bf7d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java @@ -1,7 +1,8 @@ package com.tencent.supersonic.headless.api.pojo.request; -import com.tencent.supersonic.common.pojo.DateConf; import javax.validation.constraints.NotNull; + +import com.tencent.supersonic.common.pojo.DateConf; import lombok.Data; import lombok.ToString; @@ -9,10 +10,9 @@ import lombok.ToString; @ToString public class ItemValueReq { - @NotNull - private Long id; + @NotNull private Long id; private DateConf dateConf; private Long limit = 10L; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetaBatchReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetaBatchReq.java index 10c49b372..dfd541402 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetaBatchReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetaBatchReq.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.tencent.supersonic.common.pojo.enums.EventType; import lombok.Data; + import java.util.List; @Data @@ -13,18 +14,13 @@ public class MetaBatchReq { private List modelIds; - /** - * 最后变更的状态 - */ + /** 最后变更的状态 */ private Integer status; - /** - * 批量执行分类信息 - */ + /** 批量执行分类信息 */ private EventType type; private List classifications; private Integer sensitiveLevel; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricBaseReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricBaseReq.java index f3991bb97..f001888cf 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricBaseReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricBaseReq.java @@ -10,7 +10,6 @@ import org.springframework.util.CollectionUtils; import java.util.List; import java.util.Map; - @Data public class MetricBaseReq extends SchemaItem { @@ -39,5 +38,4 @@ public class MetricBaseReq extends SchemaItem { } return StringUtils.join(classifications, ","); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java index ce8f41d54..a83899e59 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java @@ -16,5 +16,4 @@ public class MetricQueryReq { private Long limit; private List order; private boolean nativeQuery = false; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricReq.java index d25204d7d..98f77301e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricReq.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.alibaba.fastjson.JSONObject; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import lombok.Data; @Data @@ -19,9 +19,11 @@ public class MetricReq extends MetricBaseReq { public String getTypeParamsJson() { if (MetricDefineType.FIELD.equals(metricDefineType) && metricDefineByFieldParams != null) { return JSONObject.toJSONString(metricDefineByFieldParams); - } else if (MetricDefineType.MEASURE.equals(metricDefineType) && metricDefineByMeasureParams != null) { + } else if (MetricDefineType.MEASURE.equals(metricDefineType) + && metricDefineByMeasureParams != null) { return JSONObject.toJSONString(metricDefineByMeasureParams); - } else if (MetricDefineType.METRIC.equals(metricDefineType) && metricDefineByMetricParams != null) { + } else if (MetricDefineType.METRIC.equals(metricDefineType) + && metricDefineByMetricParams != null) { return JSONObject.toJSONString(metricDefineByMetricParams); } return null; @@ -29,7 +31,7 @@ public class MetricReq extends MetricBaseReq { public MetricType getMetricType() { return MetricType.isDerived(metricDefineType, metricDefineByMeasureParams) - ? MetricType.DERIVED : MetricType.ATOMIC; + ? MetricType.DERIVED + : MetricType.ATOMIC; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelReq.java index 1bcc7e91d..7b2239c27 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.request; - import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; @@ -11,7 +10,6 @@ import lombok.Data; import java.util.List; import java.util.Map; - @Data public class ModelReq extends SchemaItem { @@ -77,5 +75,4 @@ public class ModelReq extends SchemaItem { } return String.join(",", adminOrgs); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageDimensionReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageDimensionReq.java index 177d7ed33..4980cbcb2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageDimensionReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageDimensionReq.java @@ -1,12 +1,9 @@ package com.tencent.supersonic.headless.api.pojo.request; - import lombok.Data; - @Data public class PageDimensionReq extends PageSchemaItemReq { private Integer isTag; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageMetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageMetricReq.java index 8ad709659..7cba22376 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageMetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageMetricReq.java @@ -1,9 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.request; - import lombok.Data; - @Data public class PageMetricReq extends PageSchemaItemReq { @@ -12,5 +10,4 @@ public class PageMetricReq extends PageSchemaItemReq { private Integer isTag; private Integer isPublish; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageSchemaItemReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageSchemaItemReq.java index 036221cf1..f8260957a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageSchemaItemReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/PageSchemaItemReq.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.PageBaseReq; import lombok.Data; + import java.util.List; @Data diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java index 0b67f66a2..25d8baa49 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.request; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.headless.api.pojo.SchemaElement; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java index 87a17209c..bda9f7a83 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java @@ -7,11 +7,12 @@ import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.Cache; import com.tencent.supersonic.headless.api.pojo.Param; -import java.util.ArrayList; -import java.util.List; import lombok.Data; import lombok.ToString; +import java.util.ArrayList; +import java.util.List; + @Data @ToString public class QueryDataSetReq { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java index 818194b02..2ef1a3e5b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java @@ -30,14 +30,16 @@ public class QueryFilter { return false; } QueryFilter that = (QueryFilter) o; - return Objects.equal(bizName, that.bizName) && Objects.equal(name, - that.name) && operator == that.operator && Objects.equal(value, that.value) - && Objects.equal(elementID, that.elementID) && Objects.equal( - function, that.function); + return Objects.equal(bizName, that.bizName) + && Objects.equal(name, that.name) + && operator == that.operator + && Objects.equal(value, that.value) + && Objects.equal(elementID, that.elementID) + && Objects.equal(function, that.function); } @Override public int hashCode() { return Objects.hashCode(bizName, name, operator, value, elementID, function); } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java index b6d6aa028..1323254a6 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import lombok.Data; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryItemReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryItemReq.java index 63101182c..cdaa99510 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryItemReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryItemReq.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.Size; + import com.tencent.supersonic.common.pojo.DateConf; import lombok.Data; -import javax.validation.constraints.Size; import java.util.List; @Data @@ -14,7 +15,7 @@ public class QueryItemReq { private DateConf dateConf = new DateConf(); - //result size of single id + // result size of single id private Long limit = 1000L; public Long getLimit() { @@ -23,5 +24,4 @@ public class QueryItemReq { } return limit; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java index b361888b7..411836d7a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java @@ -3,10 +3,11 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; -import java.util.List; import lombok.Data; import lombok.ToString; +import java.util.List; + @Data @ToString public class QueryMetricReq { @@ -28,5 +29,4 @@ public class QueryMetricReq { private Long limit = 2000L; private boolean innerLayerNative = false; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java index 5e853c099..db5f0fc7e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java @@ -36,5 +36,4 @@ public class QueryMultiStructReq extends SemanticQueryReq { } return this.getQueryStructReqs().get(0).getCacheInfo(); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleFilter.java index 478b71ee9..93c2364d8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleFilter.java @@ -12,5 +12,4 @@ public class QueryRuleFilter { private List ruleIds; private List dataSetIds; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleReq.java index 95a4fadec..a5b3be1b2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryRuleReq.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.headless.api.pojo.ActionInfo; import com.tencent.supersonic.headless.api.pojo.RuleInfo; import com.tencent.supersonic.headless.api.pojo.SchemaItem; @@ -7,7 +9,6 @@ import com.tencent.supersonic.headless.api.pojo.enums.QueryRuleType; import lombok.Data; import lombok.ToString; -import javax.validation.constraints.NotNull; import java.util.HashMap; import java.util.Map; @@ -15,36 +16,21 @@ import java.util.Map; @ToString public class QueryRuleReq extends SchemaItem { - /** - * dataSetID - */ + /** dataSetID */ private Long dataSetId; - /** - * 规则的优先级, 1-低,2-中,3-高 - */ + /** 规则的优先级, 1-低,2-中,3-高 */ private Integer priority = 1; - /** - * 规则类型 - */ - @NotNull - private QueryRuleType ruleType; + /** 规则类型 */ + @NotNull private QueryRuleType ruleType; - /** - * 具体规则信息 - */ - @NotNull - private RuleInfo rule; + /** 具体规则信息 */ + @NotNull private RuleInfo rule; - /** - * 规则输出信息 - */ + /** 规则输出信息 */ private ActionInfo action; - - /** - * 扩展信息 - */ + /** 扩展信息 */ private Map ext = new HashMap<>(); -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java index 506c3cb68..99d4b06f6 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java @@ -23,16 +23,11 @@ public class QuerySqlReq extends SemanticQueryReq { @Override public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"dataSetId\":") - .append(dataSetId); - stringBuilder.append("\"modelIds\":") - .append(modelIds); - stringBuilder.append(",\"params\":") - .append(params); - stringBuilder.append(",\"cacheInfo\":") - .append(cacheInfo); - stringBuilder.append(",\"sql\":") - .append(sql); + stringBuilder.append("\"dataSetId\":").append(dataSetId); + stringBuilder.append("\"modelIds\":").append(modelIds); + stringBuilder.append(",\"params\":").append(params); + stringBuilder.append(",\"cacheInfo\":").append(cacheInfo); + stringBuilder.append(",\"sql\":").append(sql); stringBuilder.append('}'); return stringBuilder.toString(); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlsReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlsReq.java index d91f0db2a..03f760c92 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlsReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlsReq.java @@ -13,18 +13,12 @@ public class QuerySqlsReq extends SemanticQueryReq { @Override public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"dataSetId\":") - .append(dataSetId); - stringBuilder.append("\"modelIds\":") - .append(modelIds); - stringBuilder.append(",\"params\":") - .append(params); - stringBuilder.append(",\"cacheInfo\":") - .append(cacheInfo); - stringBuilder.append(",\"sqls\":") - .append(sqls); + stringBuilder.append("\"dataSetId\":").append(dataSetId); + stringBuilder.append("\"modelIds\":").append(modelIds); + stringBuilder.append(",\"params\":").append(params); + stringBuilder.append(",\"cacheInfo\":").append(cacheInfo); + stringBuilder.append(",\"sqls\":").append(sqls); stringBuilder.append('}'); return stringBuilder.toString(); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index e575fdc11..ecfdb930f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.request; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.pojo.Aggregator; @@ -39,7 +38,6 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; - @Data @Slf4j public class QueryStructReq extends SemanticQueryReq { @@ -56,7 +54,10 @@ public class QueryStructReq extends SemanticQueryReq { public List getGroups() { if (!CollectionUtils.isEmpty(this.groups)) { - this.groups = groups.stream().filter(group -> !StringUtils.isEmpty(group)).collect(Collectors.toList()); + this.groups = + groups.stream() + .filter(group -> !StringUtils.isEmpty(group)) + .collect(Collectors.toList()); } if (CollectionUtils.isEmpty(this.groups)) { @@ -83,24 +84,15 @@ public class QueryStructReq extends SemanticQueryReq { public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"dataSetId\":") - .append(dataSetId); - stringBuilder.append("\"modelIds\":") - .append(modelIds); - stringBuilder.append(",\"groups\":") - .append(groups); - stringBuilder.append(",\"aggregators\":") - .append(aggregators); - stringBuilder.append(",\"orders\":") - .append(orders); - stringBuilder.append(",\"filters\":") - .append(dimensionFilters); - stringBuilder.append(",\"dateInfo\":") - .append(dateInfo); - stringBuilder.append(",\"params\":") - .append(params); - stringBuilder.append(",\"limit\":") - .append(limit); + stringBuilder.append("\"dataSetId\":").append(dataSetId); + stringBuilder.append("\"modelIds\":").append(modelIds); + stringBuilder.append(",\"groups\":").append(groups); + stringBuilder.append(",\"aggregators\":").append(aggregators); + stringBuilder.append(",\"orders\":").append(orders); + stringBuilder.append(",\"filters\":").append(dimensionFilters); + stringBuilder.append(",\"dateInfo\":").append(dateInfo); + stringBuilder.append(",\"params\":").append(params); + stringBuilder.append(",\"limit\":").append(limit); stringBuilder.append('}'); return stringBuilder.toString(); } @@ -116,28 +108,17 @@ public class QueryStructReq extends SemanticQueryReq { @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"dataSetId\":") - .append(dataSetId); - sb.append("\"modelIds\":") - .append(modelIds); - sb.append(",\"groups\":") - .append(groups); - sb.append(",\"aggregators\":") - .append(aggregators); - sb.append(",\"orders\":") - .append(orders); - sb.append(",\"dimensionFilters\":") - .append(dimensionFilters); - sb.append(",\"metricFilters\":") - .append(metricFilters); - sb.append(",\"params\":") - .append(params); - sb.append(",\"dateInfo\":") - .append(dateInfo); - sb.append(",\"limit\":") - .append(limit); - sb.append(",\"cacheInfo\":") - .append(cacheInfo); + sb.append("\"dataSetId\":").append(dataSetId); + sb.append("\"modelIds\":").append(modelIds); + sb.append(",\"groups\":").append(groups); + sb.append(",\"aggregators\":").append(aggregators); + sb.append(",\"orders\":").append(orders); + sb.append(",\"dimensionFilters\":").append(dimensionFilters); + sb.append(",\"metricFilters\":").append(metricFilters); + sb.append(",\"params\":").append(params); + sb.append(",\"dateInfo\":").append(dateInfo); + sb.append(",\"limit\":").append(limit); + sb.append(",\"cacheInfo\":").append(cacheInfo); sb.append('}'); return sb.toString(); } @@ -214,7 +195,8 @@ public class QueryStructReq extends SemanticQueryReq { return selectItems; } - private SelectItem buildAggregatorSelectItem(Aggregator aggregator, QueryStructReq queryStructReq) { + private SelectItem buildAggregatorSelectItem( + Aggregator aggregator, QueryStructReq queryStructReq) { String columnName = aggregator.getColumn(); if (queryStructReq.getQueryType().isNativeAggQuery()) { return new SelectItem(new Column(columnName)); @@ -231,7 +213,10 @@ public class QueryStructReq extends SemanticQueryReq { } function.setParameters(new ExpressionList(new Column(columnName))); SelectItem selectExpressionItem = new SelectItem(function); - String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName; + String alias = + StringUtils.isNotBlank(aggregator.getAlias()) + ? aggregator.getAlias() + : columnName; selectExpressionItem.setAlias(new Alias(alias)); return selectExpressionItem; } @@ -280,7 +265,8 @@ public class QueryStructReq extends SemanticQueryReq { private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName) throws JSQLParserException { SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class); - String whereClause = sqlFilterUtils.getWhereClause(queryStructReq.getDimensionFilters(), isBizName); + String whereClause = + sqlFilterUtils.getWhereClause(queryStructReq.getDimensionFilters(), isBizName); if (StringUtils.isNotBlank(whereClause)) { Expression expression = CCJSqlParserUtil.parseCondExpression(whereClause); @@ -307,5 +293,4 @@ public class QueryStructReq extends SemanticQueryReq { } return Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_"); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java index 93721999c..e9e7270c6 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java @@ -33,24 +33,15 @@ public class QueryTagReq extends SemanticQueryReq { @Override public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"dataSetId\":") - .append(dataSetId); - stringBuilder.append("\"modelIds\":") - .append(modelIds); - stringBuilder.append(",\"groups\":") - .append(groups); - stringBuilder.append(",\"aggregators\":") - .append(aggregators); - stringBuilder.append(",\"orders\":") - .append(orders); - stringBuilder.append(",\"tagFilters\":") - .append(tagFilters); - stringBuilder.append(",\"dateInfo\":") - .append(dateInfo); - stringBuilder.append(",\"params\":") - .append(params); - stringBuilder.append(",\"limit\":") - .append(limit); + stringBuilder.append("\"dataSetId\":").append(dataSetId); + stringBuilder.append("\"modelIds\":").append(modelIds); + stringBuilder.append(",\"groups\":").append(groups); + stringBuilder.append(",\"aggregators\":").append(aggregators); + stringBuilder.append(",\"orders\":").append(orders); + stringBuilder.append(",\"tagFilters\":").append(tagFilters); + stringBuilder.append(",\"dateInfo\":").append(dateInfo); + stringBuilder.append(",\"params\":").append(params); + stringBuilder.append(",\"limit\":").append(limit); stringBuilder.append('}'); return stringBuilder.toString(); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java index b806af12c..b895689f1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java @@ -11,5 +11,4 @@ public class SchemaFilterReq { private Long dataSetId; private List modelIds = Lists.newArrayList(); - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaItemQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaItemQueryReq.java index 13b45af7a..eeef6554a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaItemQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaItemQueryReq.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.common.pojo.enums.TypeEnums; import lombok.Data; -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; + import java.util.List; @Data @@ -12,8 +14,7 @@ public class SchemaItemQueryReq { @NotEmpty(message = "id个数不可为空") private List ids; - //METRIC, DIMENSION + // METRIC, DIMENSION @NotNull(message = "type不可为空") private TypeEnums type; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java index ebaaf3a40..41fb26d6e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java @@ -13,7 +13,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - @Data @Slf4j public abstract class SemanticQueryReq { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java index fee5d077d..5cc530153 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; import com.tencent.supersonic.headless.api.pojo.SqlVariable; import lombok.Data; import org.apache.commons.lang3.StringUtils; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; + import java.util.List; @Data @@ -28,5 +29,4 @@ public class SqlExecuteReq { } return String.format(LIMIT_WRAPPER, sql, limit); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagBatchCreateReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagBatchCreateReq.java index ff0dfcba6..3bd3186a2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagBatchCreateReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagBatchCreateReq.java @@ -1,19 +1,17 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import lombok.Data; import lombok.ToString; -import javax.validation.constraints.NotNull; import java.util.List; @ToString @Data public class TagBatchCreateReq { - @NotNull - private Long modelId; + @NotNull private Long modelId; private SchemaElementType type; private List itemIds; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagDeleteReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagDeleteReq.java index 9761de1a4..c50c1c553 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagDeleteReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagDeleteReq.java @@ -11,4 +11,4 @@ public class TagDeleteReq { private List ids; private List itemIds; private TagDefineType tagDefineType; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagFilterPageReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagFilterPageReq.java index 952646185..4ec1fda61 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagFilterPageReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagFilterPageReq.java @@ -9,4 +9,4 @@ public class TagFilterPageReq extends PageSchemaItemReq { private Long tagObjectId; private TagDefineType tagDefineType; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagObjectReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagObjectReq.java index 54e8515c6..a900bed59 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagObjectReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagObjectReq.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import lombok.Data; -import javax.validation.constraints.NotNull; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -12,11 +13,10 @@ import java.util.Objects; @Data public class TagObjectReq extends SchemaItem { - @NotNull - private Long domainId; + @NotNull private Long domainId; private Map ext = new HashMap<>(); public String getExtJson() { return Objects.nonNull(ext) && ext.size() > 0 ? JSONObject.toJSONString(ext) : ""; } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java index 2c7f04851..135a1a732 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java @@ -1,20 +1,17 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; import lombok.Data; -import javax.validation.constraints.NotNull; - @Data public class TagReq extends RecordInfo { private Long id; - @NotNull - private TagDefineType tagDefineType; - - @NotNull - private Long itemId; + @NotNull private TagDefineType tagDefineType; + @NotNull private Long itemId; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TermReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TermReq.java index 26d90ae14..912d1e49a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TermReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TermReq.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; +import javax.validation.constraints.NotNull; + import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; -import javax.validation.constraints.NotNull; import java.util.List; @Data @@ -24,5 +25,4 @@ public class TermReq extends RecordInfo { private List relatedMetrics = Lists.newArrayList(); private List relateDimensions = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppDetailResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppDetailResp.java index 62f76fba7..a50b120c7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppDetailResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppDetailResp.java @@ -6,5 +6,4 @@ import lombok.Data; public class AppDetailResp extends AppResp { private String appSecret; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java index ce7998942..d04f23336 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java @@ -1,11 +1,10 @@ package com.tencent.supersonic.headless.api.pojo.response; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; -import com.tencent.supersonic.headless.api.pojo.enums.AppStatus; import com.tencent.supersonic.headless.api.pojo.AppConfig; import com.tencent.supersonic.headless.api.pojo.Item; +import com.tencent.supersonic.headless.api.pojo.enums.AppStatus; import lombok.Data; import org.apache.commons.lang3.StringUtils; @@ -52,5 +51,4 @@ public class AppResp extends RecordInfo { } return itemSet; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java index 9971fb72f..d0c791e61 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java @@ -14,6 +14,4 @@ public class CanvasSchemaResp { private List metrics; private List dimensions; - - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ClassResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ClassResp.java index 0c16e0bf9..2edd7556c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ClassResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ClassResp.java @@ -20,20 +20,16 @@ public class ClassResp { private String fullPath; - /** - * 分类状态 - */ + /** 分类状态 */ private Integer status; - /** - * METRIC、DIMENSION、TAG - */ + /** METRIC、DIMENSION、TAG */ private String type; + private List itemIds; private Date createdAt; private String createdBy; private Date updatedAt; private String updatedBy; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetMapInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetMapInfo.java index e32182092..805b5aedd 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetMapInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetMapInfo.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import lombok.Data; + import java.util.List; @Data @@ -15,5 +16,4 @@ public class DataSetMapInfo { private List mapFields = Lists.newArrayList(); private List topFields = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java index aaf50afe0..f8f9d0620 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java @@ -35,31 +35,37 @@ public class DataSetResp extends SchemaItem { private List allDimensions = new ArrayList<>(); public List metricIds() { - return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics) - .flatMap(Collection::stream).collect(Collectors.toList()); + return getDataSetModelConfigs().stream() + .map(DataSetModelConfig::getMetrics) + .flatMap(Collection::stream) + .collect(Collectors.toList()); } public List dimensionIds() { - return getDataSetModelConfigs().stream().map(DataSetModelConfig::getDimensions) - .flatMap(Collection::stream).collect(Collectors.toList()); + return getDataSetModelConfigs().stream() + .map(DataSetModelConfig::getDimensions) + .flatMap(Collection::stream) + .collect(Collectors.toList()); } public List getAllModels() { - return getDataSetModelConfigs().stream().map(DataSetModelConfig::getId) + return getDataSetModelConfigs().stream() + .map(DataSetModelConfig::getId) .collect(Collectors.toList()); } public List getAllIncludeAllModels() { - return getDataSetModelConfigs().stream().filter(DataSetModelConfig::getIncludesAll) + return getDataSetModelConfigs().stream() + .filter(DataSetModelConfig::getIncludesAll) .map(DataSetModelConfig::getId) .collect(Collectors.toList()); } private List getDataSetModelConfigs() { - if (dataSetDetail == null || CollectionUtils.isEmpty(dataSetDetail.getDataSetModelConfigs())) { + if (dataSetDetail == null + || CollectionUtils.isEmpty(dataSetDetail.getDataSetModelConfigs())) { return Lists.newArrayList(); } return dataSetDetail.getDataSetModelConfigs(); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java index 532b8e181..84345580c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java @@ -33,4 +33,4 @@ public class DataSetSchemaResp extends DataSetResp { } return null; } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java index 6ef425172..e60fb122c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - @Data @AllArgsConstructor @NoArgsConstructor @@ -65,5 +64,4 @@ public class DatabaseResp extends RecordInfo { } return ""; } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java index e3ffff340..f2ea41736 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java @@ -1,13 +1,14 @@ package com.tencent.supersonic.headless.api.pojo.response; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import javax.validation.constraints.NotNull; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; -import javax.validation.constraints.NotNull; import lombok.Data; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; + @Data public class DictItemResp { @@ -17,19 +18,12 @@ public class DictItemResp { private String bizName; - @NotNull - private TypeEnums type; - @NotNull - private Long itemId; + @NotNull private TypeEnums type; + @NotNull private Long itemId; private ItemValueConfig config; - /** - * ONLINE - 正常更新 - * OFFLINE - 停止更新,但字典文件不删除 - * DELETED - 停止更新,且删除字典文件 - */ - @NotNull - private StatusEnum status; + /** ONLINE - 正常更新 OFFLINE - 停止更新,但字典文件不删除 DELETED - 停止更新,且删除字典文件 */ + @NotNull private StatusEnum status; public String getNature() { return UNDERLINE + modelId + UNDERLINE + itemId; @@ -38,5 +32,4 @@ public class DictItemResp { public String fetchDictFileName() { return String.format("dic_value_%d_%s_%s", modelId, type.name(), itemId); } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java index cee376e44..97c85047d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.response; - import lombok.Data; import lombok.ToString; @@ -16,5 +15,4 @@ public class DictTaskResp extends DictItemResp { private Date createdAt; private String createdBy; private Long elapsedMs; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java index e7f55d6cc..90a56c4fe 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java @@ -15,4 +15,4 @@ public class DictValueResp { private String nature; private Long frequency; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java index 83b779b05..28107c44a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java @@ -12,5 +12,4 @@ public class DimSchemaResp extends DimensionResp { private Long useCnt = 0L; private List entityAlias; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java index 80369ef86..ea7692b7f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.response; - import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.SchemaItem; @@ -12,7 +11,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - @Data @ToString(callSuper = true) public class DimensionResp extends SchemaItem { @@ -28,7 +26,7 @@ public class DimensionResp extends SchemaItem { private String modelBizName; private String modelFilterSql; - //DATE ID CATEGORY + // DATE ID CATEGORY private String semanticType; private String alias; @@ -50,5 +48,4 @@ public class DimensionResp extends SchemaItem { public boolean isPartitionTime() { return DimensionType.isPartitionTime(type); } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemQueryResultResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemQueryResultResp.java index d41793a34..fb32be0e0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemQueryResultResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemQueryResultResp.java @@ -11,5 +11,4 @@ import java.util.List; public class ItemQueryResultResp { private List results; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemResp.java index 06d4afcf2..3f1e824cb 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemResp.java @@ -33,5 +33,4 @@ public class ItemResp { public boolean isRoot() { return parentId == null || parentId == 0; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemUseResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemUseResp.java index 933a0eba2..8ebf96fba 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemUseResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemUseResp.java @@ -21,4 +21,4 @@ public class ItemUseResp { this.bizName = bizName; this.useCnt = useCnt; } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java index 7f27868a5..dead63f87 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java @@ -15,4 +15,4 @@ public class ItemValueResp { private String name; private String bizName; private List valueDistributionList; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java index 0ded888ab..cbdd64e22 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java @@ -15,5 +15,4 @@ public class MapInfoResp { private Map dataSetMapInfo = new HashMap<>(); private Map> terms = new HashMap<>(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MeasureResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MeasureResp.java index 5c1941400..7e3177032 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MeasureResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MeasureResp.java @@ -2,13 +2,12 @@ package com.tencent.supersonic.headless.api.pojo.response; import lombok.Data; - @Data public class MeasureResp { private String name; - //sum max min avg count distinct + // sum max min avg count distinct private String agg; private String expr; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java index 26626de6c..0ad44c364 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java @@ -1,15 +1,14 @@ package com.tencent.supersonic.headless.api.pojo.response; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.RelateDimension; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; +import com.tencent.supersonic.headless.api.pojo.RelateDimension; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import lombok.Data; import lombok.ToString; import org.apache.commons.collections.CollectionUtils; @@ -21,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - @Data @ToString(callSuper = true) public class MetricResp extends SchemaItem { @@ -34,7 +32,7 @@ public class MetricResp extends SchemaItem { private String modelName; - //ATOMIC DERIVED + // ATOMIC DERIVED private String type; private String dataFormatType; @@ -80,7 +78,8 @@ public class MetricResp extends SchemaItem { } public String getRelaDimensionIdKey() { - if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { + if (relateDimension == null + || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { return ""; } return relateDimension.getDrillDownDimensions().stream() @@ -90,7 +89,8 @@ public class MetricResp extends SchemaItem { } public List getDrillDownDimensions() { - if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { + if (relateDimension == null + || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { return Lists.newArrayList(); } return relateDimension.getDrillDownDimensions(); @@ -106,5 +106,4 @@ public class MetricResp extends SchemaItem { } return ""; } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java index 117a4d2f9..a6e334e05 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java @@ -8,5 +8,4 @@ import lombok.ToString; public class MetricSchemaResp extends MetricResp { private Long useCnt = 0L; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java index a8e7c3cff..3cc475def 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java @@ -89,8 +89,10 @@ public class ModelResp extends SchemaItem { return fieldSet; } if (!CollectionUtils.isEmpty(modelDetail.getFields())) { - fieldSet.addAll(modelDetail.getFields().stream() - .map(Field::getFieldName).collect(Collectors.toSet())); + fieldSet.addAll( + modelDetail.getFields().stream() + .map(Field::getFieldName) + .collect(Collectors.toSet())); } return fieldSet; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java index 8a48e2f68..c24d9ca17 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java @@ -25,12 +25,12 @@ public class ModelSchemaResp extends ModelResp { return Sets.newHashSet(); } else { Set modelClusterSet = new HashSet(); - this.modelRelas.forEach((modelRela) -> { - modelClusterSet.add(modelRela.getToModelId()); - modelClusterSet.add(modelRela.getFromModelId()); - }); + this.modelRelas.forEach( + (modelRela) -> { + modelClusterSet.add(modelRela.getToModelId()); + modelClusterSet.add(modelRela.getFromModelId()); + }); return modelClusterSet; } } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java index 832f5535d..367ab63ff 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java @@ -29,9 +29,10 @@ public class ParseResp { } public List getSelectedParses() { - selectedParses = selectedParses.stream() - .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed()) - .collect(Collectors.toList()); + selectedParses = + selectedParses.stream() + .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed()) + .collect(Collectors.toList()); generateParseInfoId(selectedParses); return selectedParses; } @@ -42,5 +43,4 @@ public class ParseResp { parseInfo.setId(i + 1); } } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryRuleResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryRuleResp.java index f256a75cf..2814f3919 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryRuleResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryRuleResp.java @@ -1,48 +1,33 @@ package com.tencent.supersonic.headless.api.pojo.response; +import javax.validation.constraints.NotNull; + import com.tencent.supersonic.headless.api.pojo.ActionInfo; import com.tencent.supersonic.headless.api.pojo.RuleInfo; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.QueryRuleType; import lombok.Data; -import javax.validation.constraints.NotNull; import java.util.Map; @Data public class QueryRuleResp extends SchemaItem { - /** - * dataSetID - */ + /** dataSetID */ private Long dataSetId; - /** - * 规则的优先级, 1-低,2-中,3-高 - */ + /** 规则的优先级, 1-低,2-中,3-高 */ private Integer priority = 1; - /** - * 规则类型 - */ - @NotNull - private QueryRuleType ruleType; + /** 规则类型 */ + @NotNull private QueryRuleType ruleType; - /** - * 具体规则信息 - */ - @NotNull - private RuleInfo rule; + /** 具体规则信息 */ + @NotNull private RuleInfo rule; - /** - * 规则输出信息 - */ + /** 规则输出信息 */ private ActionInfo action; - - /** - * 扩展信息 - */ + /** 扩展信息 */ private Map ext; - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java index 80e6be4f6..2f2be7d5c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java @@ -13,8 +13,7 @@ public class S2Term { public int offset; public int frequency = 0; - public S2Term() { - } + public S2Term() {} public S2Term(String word, Nature nature) { this.word = word; @@ -47,4 +46,4 @@ public class S2Term { } return super.equals(obj); } -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java index da67ed3ac..ea361bcc4 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.headless.api.pojo.response; -import java.util.List; import lombok.Data; +import java.util.List; @Data public class SearchResp { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java index 742abaa4f..c80008df2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java @@ -35,8 +35,8 @@ public class SearchResult { return false; } SearchResult searchResult1 = (SearchResult) o; - return Objects.equals(recommend, searchResult1.recommend) && Objects.equals(modelName, - searchResult1.modelName); + return Objects.equals(recommend, searchResult1.recommend) + && Objects.equals(modelName, searchResult1.modelName); } @Override diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticQueryResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticQueryResp.java index d8888279b..183ba8d7a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticQueryResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticQueryResp.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.response; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.QueryAuthorization; import com.tencent.supersonic.common.pojo.QueryColumn; @@ -8,6 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.QueryResult; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import lombok.Data; import lombok.ToString; + import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -29,7 +29,9 @@ public class SemanticQueryResp extends QueryResult> { public List getDimensionColumns() { return columns.stream() - .filter(queryColumn -> !SemanticType.NUMBER.name().equals(queryColumn.getShowType())) + .filter( + queryColumn -> + !SemanticType.NUMBER.name().equals(queryColumn.getShowType())) .collect(Collectors.toList()); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index 6477e08f1..df30a0efa 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -37,27 +37,34 @@ public class SemanticSchemaResp { return String.format("%s_%s", schemaType, StringUtils.join(modelIds, UNDERLINE)); } return String.format("%s_%s", schemaType, dataSetId); - } public MetricSchemaResp getMetric(String bizName) { - return metrics.stream().filter(metric -> bizName.equalsIgnoreCase(metric.getBizName())) - .findFirst().orElse(null); + return metrics.stream() + .filter(metric -> bizName.equalsIgnoreCase(metric.getBizName())) + .findFirst() + .orElse(null); } public MetricSchemaResp getMetric(Long id) { - return metrics.stream().filter(metric -> id.equals(metric.getId())) - .findFirst().orElse(null); + return metrics.stream() + .filter(metric -> id.equals(metric.getId())) + .findFirst() + .orElse(null); } public DimSchemaResp getDimension(String bizName) { - return dimensions.stream().filter(dimension -> bizName.equalsIgnoreCase(dimension.getBizName())) - .findFirst().orElse(null); + return dimensions.stream() + .filter(dimension -> bizName.equalsIgnoreCase(dimension.getBizName())) + .findFirst() + .orElse(null); } public DimSchemaResp getDimension(Long id) { - return dimensions.stream().filter(dimension -> id.equals(dimension.getId())) - .findFirst().orElse(null); + return dimensions.stream() + .filter(dimension -> id.equals(dimension.getId())) + .findFirst() + .orElse(null); } public Set getNameFromBizNames(Set bizNames) { @@ -75,5 +82,4 @@ public class SemanticSchemaResp { } return names; } - -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticTranslateResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticTranslateResp.java index 3a643b07b..f3a94c3f9 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticTranslateResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticTranslateResp.java @@ -20,5 +20,4 @@ public class SemanticTranslateResp implements Serializable { private boolean isOk; private String errMsg; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagItem.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagItem.java index d96cfc0e1..42e9f4cfe 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagItem.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagItem.java @@ -8,5 +8,4 @@ public class TagItem { private Long itemId; private int isTag; - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagObjectResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagObjectResp.java index 5af318a6c..699bef8de 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagObjectResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagObjectResp.java @@ -10,13 +10,9 @@ import java.util.Map; @ToString(callSuper = true) public class TagObjectResp extends SchemaItem { - /** - * 关联到某个主题域下 - */ + /** 关联到某个主题域下 */ private Long domainId; - /** - * 扩展信息 - */ + /** 扩展信息 */ private Map ext; -} \ No newline at end of file +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java index 7e5ff960a..d6846c502 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java @@ -42,5 +42,4 @@ public class TagResp extends RecordInfo { private Integer sensitiveLevel; private Map ext = new HashMap(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TermResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TermResp.java index 7b84cc421..fb5927b47 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TermResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TermResp.java @@ -26,5 +26,4 @@ public class TermResp extends RecordInfo { private List relatedMetrics = Lists.newArrayList(); private List relateDimensions = Lists.newArrayList(); - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/UnAvailableItemResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/UnAvailableItemResp.java index 7553f8be4..93b79fe16 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/UnAvailableItemResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/UnAvailableItemResp.java @@ -12,5 +12,4 @@ public class UnAvailableItemResp { private List metricResps; private List dimensionResps; - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/ChatQueryContext.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/ChatQueryContext.java index e9229450c..1ddb40255 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/ChatQueryContext.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/ChatQueryContext.java @@ -41,17 +41,14 @@ public class ChatQueryContext { private Map> modelIdToDataSetIds; private User user; private boolean saveAnswer; - @Builder.Default - private Text2SQLType text2SQLType = Text2SQLType.RULE_AND_LLM; + @Builder.Default private Text2SQLType text2SQLType = Text2SQLType.RULE_AND_LLM; private QueryFilters queryFilters; private List candidateQueries = new ArrayList<>(); private SchemaMapInfo mapInfo = new SchemaMapInfo(); private SemanticParseInfo contextParseInfo; private MapModeEnum mapModeEnum = MapModeEnum.STRICT; - @JsonIgnore - private SemanticSchema semanticSchema; - @JsonIgnore - private ChatWorkflowState chatWorkflowState; + @JsonIgnore private SemanticSchema semanticSchema; + @JsonIgnore private ChatWorkflowState chatWorkflowState; private QueryDataType queryDataType = QueryDataType.ALL; private ChatModelConfig modelConfig; private PromptConfig promptConfig; @@ -59,12 +56,16 @@ public class ChatQueryContext { public List getCandidateQueries() { ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class); - int parseShowCount = Integer.parseInt(parserConfig.getParameterValue(ParserConfig.PARSER_SHOW_COUNT)); - candidateQueries = candidateQueries.stream() - .sorted(Comparator.comparing(semanticQuery -> semanticQuery.getParseInfo().getScore(), - Comparator.reverseOrder())) - .limit(parseShowCount) - .collect(Collectors.toList()); + int parseShowCount = + Integer.parseInt(parserConfig.getParameterValue(ParserConfig.PARSER_SHOW_COUNT)); + candidateQueries = + candidateQueries.stream() + .sorted( + Comparator.comparing( + semanticQuery -> semanticQuery.getParseInfo().getScore(), + Comparator.reverseOrder())) + .limit(parseShowCount) + .collect(Collectors.toList()); return candidateQueries; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/AggCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/AggCorrector.java index a026726d7..e605dcd85 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/AggCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/AggCorrector.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.corrector; - import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; @@ -9,9 +8,7 @@ import org.springframework.util.CollectionUtils; import java.util.List; -/** - * Verify whether the SQL aggregate function is missing. If it is missing, fill it in. - */ +/** Verify whether the SQL aggregate function is missing. If it is missing, fill it in. */ @Slf4j public class AggCorrector extends BaseSemanticCorrector { @@ -20,13 +17,14 @@ public class AggCorrector extends BaseSemanticCorrector { addAggregate(chatQueryContext, semanticParseInfo); } - private void addAggregate(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { - List sqlGroupByFields = SqlSelectHelper.getGroupByFields( - semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); + private void addAggregate( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + List sqlGroupByFields = + SqlSelectHelper.getGroupByFields( + semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); if (CollectionUtils.isEmpty(sqlGroupByFields)) { return; } addAggregateToMetric(chatQueryContext, semanticParseInfo); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java index 5ee422cda..813d781a6 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java @@ -23,8 +23,8 @@ import java.util.Set; import java.util.stream.Collectors; /** - * basic semantic correction functionality, offering common methods and an - * abstract method called doCorrect + * basic semantic correction functionality, offering common methods and an abstract method called + * doCorrect */ @Slf4j public abstract class BaseSemanticCorrector implements SemanticCorrector { @@ -35,15 +35,20 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { return; } doCorrect(chatQueryContext, semanticParseInfo); - log.debug("sqlCorrection:{} sql:{}", this.getClass().getSimpleName(), semanticParseInfo.getSqlInfo()); + log.debug( + "sqlCorrection:{} sql:{}", + this.getClass().getSimpleName(), + semanticParseInfo.getSqlInfo()); } catch (Exception e) { log.error(String.format("correct error,sqlInfo:%s", semanticParseInfo.getSqlInfo()), e); } } - public abstract void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo); + public abstract void doCorrect( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo); - protected Map getFieldNameMap(ChatQueryContext chatQueryContext, Long dataSetId) { + protected Map getFieldNameMap( + ChatQueryContext chatQueryContext, Long dataSetId) { Map result = getFieldNameMapFromDB(chatQueryContext, dataSetId); if (chatQueryContext.containsPartitionDimensions(dataSetId)) { @@ -58,7 +63,8 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { return result; } - private static Map getFieldNameMapFromDB(ChatQueryContext chatQueryContext, Long dataSetId) { + private static Map getFieldNameMapFromDB( + ChatQueryContext chatQueryContext, Long dataSetId) { SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); List dbAllFields = new ArrayList<>(); @@ -68,53 +74,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { // support fieldName and field alias return dbAllFields.stream() .filter(entry -> dataSetId.equals(entry.getDataSetId())) - .flatMap(schemaElement -> { - Set elements = new HashSet<>(); - elements.add(schemaElement.getName()); - if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { - elements.addAll(schemaElement.getAlias()); - } - return elements.stream(); - }) - .collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1)); - } - - protected void addAggregateToMetric(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { - //add aggregate to all metric - String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); - Long dataSetId = semanticParseInfo.getDataSet().getDataSetId(); - List metrics = getMetricElements(chatQueryContext, dataSetId); - - Map metricToAggregate = metrics.stream() - .map(schemaElement -> { - if (Objects.isNull(schemaElement.getDefaultAgg())) { - schemaElement.setDefaultAgg(AggregateTypeEnum.SUM.name()); - } - return schemaElement; - }).flatMap(schemaElement -> { - Set elements = new HashSet<>(); - elements.add(schemaElement.getName()); - if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { - elements.addAll(schemaElement.getAlias()); - } - return elements.stream().map(element -> Pair.of(element, schemaElement.getDefaultAgg()) - ); - }).collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - - if (CollectionUtils.isEmpty(metricToAggregate)) { - return; - } - String aggregateSql = SqlAddHelper.addAggregateToField(correctS2SQL, metricToAggregate); - semanticParseInfo.getSqlInfo().setCorrectedS2SQL(aggregateSql); - } - - protected List getMetricElements(ChatQueryContext chatQueryContext, Long dataSetId) { - SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); - return semanticSchema.getMetrics(dataSetId); - } - - protected Set getDimensions(Long dataSetId, SemanticSchema semanticSchema) { - Set dimensions = semanticSchema.getDimensions(dataSetId).stream() .flatMap( schemaElement -> { Set elements = new HashSet<>(); @@ -123,26 +82,88 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { elements.addAll(schemaElement.getAlias()); } return elements.stream(); - } - ).collect(Collectors.toSet()); + }) + .collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1)); + } + + protected void addAggregateToMetric( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + // add aggregate to all metric + String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); + Long dataSetId = semanticParseInfo.getDataSet().getDataSetId(); + List metrics = getMetricElements(chatQueryContext, dataSetId); + + Map metricToAggregate = + metrics.stream() + .map( + schemaElement -> { + if (Objects.isNull(schemaElement.getDefaultAgg())) { + schemaElement.setDefaultAgg(AggregateTypeEnum.SUM.name()); + } + return schemaElement; + }) + .flatMap( + schemaElement -> { + Set elements = new HashSet<>(); + elements.add(schemaElement.getName()); + if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { + elements.addAll(schemaElement.getAlias()); + } + return elements.stream() + .map( + element -> + Pair.of( + element, + schemaElement.getDefaultAgg())); + }) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + + if (CollectionUtils.isEmpty(metricToAggregate)) { + return; + } + String aggregateSql = SqlAddHelper.addAggregateToField(correctS2SQL, metricToAggregate); + semanticParseInfo.getSqlInfo().setCorrectedS2SQL(aggregateSql); + } + + protected List getMetricElements( + ChatQueryContext chatQueryContext, Long dataSetId) { + SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); + return semanticSchema.getMetrics(dataSetId); + } + + protected Set getDimensions(Long dataSetId, SemanticSchema semanticSchema) { + Set dimensions = + semanticSchema.getDimensions(dataSetId).stream() + .flatMap( + schemaElement -> { + Set elements = new HashSet<>(); + elements.add(schemaElement.getName()); + if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { + elements.addAll(schemaElement.getAlias()); + } + return elements.stream(); + }) + .collect(Collectors.toSet()); dimensions.add(TimeDimensionEnum.DAY.getChName()); return dimensions; } - protected boolean containsPartitionDimensions(ChatQueryContext chatQueryContext, - SemanticParseInfo semanticParseInfo) { + protected boolean containsPartitionDimensions( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { Long dataSetId = semanticParseInfo.getDataSetId(); SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId); return dataSetSchema.containsPartitionDimensions(); } - protected void removeDateIfExist(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + protected void removeDateIfExist( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); Set removeFieldNames = new HashSet<>(); removeFieldNames.addAll(TimeDimensionEnum.getChNameList()); removeFieldNames.addAll(TimeDimensionEnum.getNameList()); - Map fieldNameMap = getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId()); + Map fieldNameMap = + getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId()); removeFieldNames.removeIf(fieldName -> fieldNameMap.containsKey(fieldName)); if (!CollectionUtils.isEmpty(removeFieldNames)) { correctS2SQL = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java index ee795001f..f42289f15 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.chat.corrector; +import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SqlInfo; @@ -18,9 +18,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -/** - * Perform SQL corrections on the "Group by" section in S2SQL. - */ +/** Perform SQL corrections on the "Group by" section in S2SQL. */ @Slf4j public class GroupByCorrector extends BaseSemanticCorrector { @@ -33,13 +31,14 @@ public class GroupByCorrector extends BaseSemanticCorrector { addGroupByFields(chatQueryContext, semanticParseInfo); } - private Boolean needAddGroupBy(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + private Boolean needAddGroupBy( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { if (!QueryType.METRIC.equals(semanticParseInfo.getQueryType())) { return false; } Long dataSetId = semanticParseInfo.getDataSetId(); - //add dimension group by + // add dimension group by SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectedS2SQL(); SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); @@ -48,7 +47,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { log.debug("no need to add groupby ,existed distinct in s2sql:{}", correctS2SQL); return false; } - //add alias field name + // add alias field name Set dimensions = getDimensions(dataSetId, semanticSchema); List selectFields = SqlSelectHelper.getSelectFields(correctS2SQL); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) { @@ -63,33 +62,40 @@ public class GroupByCorrector extends BaseSemanticCorrector { return false; } Environment environment = ContextUtils.getBean(Environment.class); - String correctorAdditionalInfo = environment.getProperty("s2.corrector.additional.information"); - if (StringUtils.isNotBlank(correctorAdditionalInfo) && !Boolean.parseBoolean(correctorAdditionalInfo)) { + String correctorAdditionalInfo = + environment.getProperty("s2.corrector.additional.information"); + if (StringUtils.isNotBlank(correctorAdditionalInfo) + && !Boolean.parseBoolean(correctorAdditionalInfo)) { return false; } return true; } - private void addGroupByFields(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + private void addGroupByFields( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { Long dataSetId = semanticParseInfo.getDataSetId(); - //add dimension group by + // add dimension group by SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectedS2SQL(); SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); - //add alias field name + // add alias field name Set dimensions = getDimensions(dataSetId, semanticSchema); List selectFields = SqlSelectHelper.gePureSelectFields(correctS2SQL); List aggregateFields = SqlSelectHelper.getAggregateFields(correctS2SQL); - Set groupByFields = selectFields.stream() - .filter(field -> dimensions.contains(field)) - .filter(field -> { - if (!CollectionUtils.isEmpty(aggregateFields) && aggregateFields.contains(field)) { - return false; - } - return true; - }) - .collect(Collectors.toSet()); - semanticParseInfo.getSqlInfo().setCorrectedS2SQL(SqlAddHelper.addGroupBy(correctS2SQL, groupByFields)); + Set groupByFields = + selectFields.stream() + .filter(field -> dimensions.contains(field)) + .filter( + field -> { + if (!CollectionUtils.isEmpty(aggregateFields) + && aggregateFields.contains(field)) { + return false; + } + return true; + }) + .collect(Collectors.toSet()); + semanticParseInfo + .getSqlInfo() + .setCorrectedS2SQL(SqlAddHelper.addGroupBy(correctS2SQL, groupByFields)); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/HavingCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/HavingCorrector.java index 5ab7c2df0..4174dd014 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/HavingCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/HavingCorrector.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.headless.chat.corrector; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.chat.ChatQueryContext; @@ -17,25 +17,24 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -/** - * Perform SQL corrections on the "Having" section in S2SQL. - */ +/** Perform SQL corrections on the "Having" section in S2SQL. */ @Slf4j public class HavingCorrector extends BaseSemanticCorrector { @Override public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { - //add aggregate to all metric + // add aggregate to all metric addHaving(chatQueryContext, semanticParseInfo); - //decide whether add having expression field to select + // decide whether add having expression field to select Environment environment = ContextUtils.getBean(Environment.class); - String correctorAdditionalInfo = environment.getProperty("s2.corrector.additional.information"); - if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) { + String correctorAdditionalInfo = + environment.getProperty("s2.corrector.additional.information"); + if (StringUtils.isNotBlank(correctorAdditionalInfo) + && Boolean.parseBoolean(correctorAdditionalInfo)) { addHavingToSelect(semanticParseInfo); } - } private void addHaving(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { @@ -43,13 +42,16 @@ public class HavingCorrector extends BaseSemanticCorrector { SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); - Set metrics = semanticSchema.getMetrics(dataSet).stream() - .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); + Set metrics = + semanticSchema.getMetrics(dataSet).stream() + .map(schemaElement -> schemaElement.getName()) + .collect(Collectors.toSet()); if (CollectionUtils.isEmpty(metrics)) { return; } - String havingSql = SqlAddHelper.addHaving(semanticParseInfo.getSqlInfo().getCorrectedS2SQL(), metrics); + String havingSql = + SqlAddHelper.addHaving(semanticParseInfo.getSqlInfo().getCorrectedS2SQL(), metrics); semanticParseInfo.getSqlInfo().setCorrectedS2SQL(havingSql); } @@ -60,10 +62,10 @@ public class HavingCorrector extends BaseSemanticCorrector { } List havingExpressionList = SqlSelectHelper.getHavingExpression(correctS2SQL); if (!CollectionUtils.isEmpty(havingExpressionList)) { - String replaceSql = SqlAddHelper.addFunctionToSelect(correctS2SQL, havingExpressionList); + String replaceSql = + SqlAddHelper.addFunctionToSelect(correctS2SQL, havingExpressionList); semanticParseInfo.getSqlInfo().setCorrectedS2SQL(replaceSql); } return; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/S2SqlDateHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/S2SqlDateHelper.java index 98b167cdf..7d55a782f 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/S2SqlDateHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/S2SqlDateHelper.java @@ -21,7 +21,8 @@ public class S2SqlDateHelper { if (Objects.isNull(dataSetId)) { return defaultDate; } - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); if (dataSetSchema == null || dataSetSchema.getTagTypeTimeDefaultConfig() == null) { return defaultDate; } @@ -30,13 +31,14 @@ public class S2SqlDateHelper { return getDefaultDate(defaultDate, tagTypeTimeDefaultConfig, partitionTimeFormat).getLeft(); } - public static Pair getStartEndDate(ChatQueryContext chatQueryContext, Long dataSetId, - QueryType queryType) { + public static Pair getStartEndDate( + ChatQueryContext chatQueryContext, Long dataSetId, QueryType queryType) { String defaultDate = DateUtils.getBeforeDate(0); if (Objects.isNull(dataSetId)) { return Pair.of(defaultDate, defaultDate); } - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); if (Objects.isNull(dataSetSchema)) { return Pair.of(defaultDate, defaultDate); } @@ -48,9 +50,8 @@ public class S2SqlDateHelper { return getDefaultDate(defaultDate, defaultConfig, partitionTimeFormat); } - private static Pair getDefaultDate(String defaultDate, - TimeDefaultConfig defaultConfig, - String partitionTimeFormat) { + private static Pair getDefaultDate( + String defaultDate, TimeDefaultConfig defaultConfig, String partitionTimeFormat) { if (defaultConfig == null) { return Pair.of(null, null); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java index a838b59fe..50e21a374 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java @@ -1,21 +1,21 @@ package com.tencent.supersonic.headless.chat.corrector; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.DateUtils; -import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.jsqlparser.AggregateEnum; import com.tencent.supersonic.common.jsqlparser.FieldExpression; import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SqlInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; -import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.headless.chat.parser.llm.ParseResult; +import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -27,9 +27,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/** - * Perform schema corrections on the Schema information in S2SQL. - */ +/** Perform schema corrections on the Schema information in S2SQL. */ @Slf4j public class SchemaCorrector extends BaseSemanticCorrector { @@ -49,7 +47,8 @@ public class SchemaCorrector extends BaseSemanticCorrector { correctFieldName(chatQueryContext, semanticParseInfo); } - private void removeDateFields(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + private void removeDateFields( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) { return; } @@ -69,8 +68,10 @@ public class SchemaCorrector extends BaseSemanticCorrector { sqlInfo.setCorrectedS2SQL(replaceAlias); } - private void correctFieldName(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { - Map fieldNameMap = getFieldNameMap(chatQueryContext, semanticParseInfo.getDataSetId()); + private void correctFieldName( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + Map fieldNameMap = + getFieldNameMap(chatQueryContext, semanticParseInfo.getDataSetId()); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String sql = SqlReplaceHelper.replaceFields(sqlInfo.getCorrectedS2SQL(), fieldNameMap); sqlInfo.setCorrectedS2SQL(sql); @@ -82,13 +83,20 @@ public class SchemaCorrector extends BaseSemanticCorrector { return; } - Map> fieldValueToFieldNames = linking.stream().collect( - Collectors.groupingBy(LLMReq.ElementValue::getFieldValue, - Collectors.mapping(LLMReq.ElementValue::getFieldName, Collectors.toSet()))); + Map> fieldValueToFieldNames = + linking.stream() + .collect( + Collectors.groupingBy( + LLMReq.ElementValue::getFieldValue, + Collectors.mapping( + LLMReq.ElementValue::getFieldName, + Collectors.toSet()))); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String sql = SqlReplaceHelper.replaceFieldNameByValue(sqlInfo.getCorrectedS2SQL(), fieldValueToFieldNames); + String sql = + SqlReplaceHelper.replaceFieldNameByValue( + sqlInfo.getCorrectedS2SQL(), fieldValueToFieldNames); sqlInfo.setCorrectedS2SQL(sql); } @@ -111,24 +119,31 @@ public class SchemaCorrector extends BaseSemanticCorrector { return; } - Map> filedNameToValueMap = linking.stream().collect( - Collectors.groupingBy(LLMReq.ElementValue::getFieldName, - Collectors.mapping(LLMReq.ElementValue::getFieldValue, Collectors.toMap( - oldValue -> oldValue, - newValue -> newValue, - (existingValue, newValue) -> newValue) - ))); + Map> filedNameToValueMap = + linking.stream() + .collect( + Collectors.groupingBy( + LLMReq.ElementValue::getFieldName, + Collectors.mapping( + LLMReq.ElementValue::getFieldValue, + Collectors.toMap( + oldValue -> oldValue, + newValue -> newValue, + (existingValue, newValue) -> newValue)))); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String sql = SqlReplaceHelper.replaceValue(sqlInfo.getCorrectedS2SQL(), filedNameToValueMap, false); + String sql = + SqlReplaceHelper.replaceValue( + sqlInfo.getCorrectedS2SQL(), filedNameToValueMap, false); sqlInfo.setCorrectedS2SQL(sql); } - public void removeFilterIfNotInLinkingValue(ChatQueryContext chatQueryContext, - SemanticParseInfo semanticParseInfo) { + public void removeFilterIfNotInLinkingValue( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectedS2SQL(); - List whereExpressionList = SqlSelectHelper.getWhereExpressions(correctS2SQL); + List whereExpressionList = + SqlSelectHelper.getWhereExpressions(correctS2SQL); if (CollectionUtils.isEmpty(whereExpressionList)) { return; } @@ -139,20 +154,39 @@ public class SchemaCorrector extends BaseSemanticCorrector { if (CollectionUtils.isEmpty(linkingValues)) { linkingValues = new ArrayList<>(); } - Set linkingFieldNames = linkingValues.stream().map(linking -> linking.getFieldName()) - .collect(Collectors.toSet()); + Set linkingFieldNames = + linkingValues.stream() + .map(linking -> linking.getFieldName()) + .collect(Collectors.toSet()); - Set removeFieldNames = whereExpressionList.stream() - .filter(fieldExpression -> StringUtils.isBlank(fieldExpression.getFunction())) - .filter(fieldExpression -> !TimeDimensionEnum.containsTimeDimension(fieldExpression.getFieldName())) - .filter(fieldExpression -> FilterOperatorEnum.EQUALS.getValue().equals(fieldExpression.getOperator())) - .filter(fieldExpression -> dimensions.contains(fieldExpression.getFieldName())) - .filter(fieldExpression -> !DateUtils.isAnyDateString(fieldExpression.getFieldValue().toString())) - .filter(fieldExpression -> !linkingFieldNames.contains(fieldExpression.getFieldName())) - .map(fieldExpression -> fieldExpression.getFieldName()).collect(Collectors.toSet()); + Set removeFieldNames = + whereExpressionList.stream() + .filter( + fieldExpression -> + StringUtils.isBlank(fieldExpression.getFunction())) + .filter( + fieldExpression -> + !TimeDimensionEnum.containsTimeDimension( + fieldExpression.getFieldName())) + .filter( + fieldExpression -> + FilterOperatorEnum.EQUALS + .getValue() + .equals(fieldExpression.getOperator())) + .filter( + fieldExpression -> + dimensions.contains(fieldExpression.getFieldName())) + .filter( + fieldExpression -> + !DateUtils.isAnyDateString( + fieldExpression.getFieldValue().toString())) + .filter( + fieldExpression -> + !linkingFieldNames.contains(fieldExpression.getFieldName())) + .map(fieldExpression -> fieldExpression.getFieldName()) + .collect(Collectors.toSet()); String sql = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames); sqlInfo.setCorrectedS2SQL(sql); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java index 88bfe17a1..ff31d7bf0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.corrector; - import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; @@ -23,9 +22,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/** - * Perform SQL corrections on the "Select" section in S2SQL. - */ +/** Perform SQL corrections on the "Select" section in S2SQL. */ @Slf4j public class SelectCorrector extends BaseSemanticCorrector { @@ -36,7 +33,8 @@ public class SelectCorrector extends BaseSemanticCorrector { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); List aggregateFields = SqlSelectHelper.getAggregateFields(correctS2SQL); List selectFields = SqlSelectHelper.getSelectFields(correctS2SQL); - // If the number of aggregated fields is equal to the number of queried fields, do not add fields to select. + // If the number of aggregated fields is equal to the number of queried fields, do not add + // fields to select. if (!CollectionUtils.isEmpty(aggregateFields) && !CollectionUtils.isEmpty(selectFields) && aggregateFields.size() == selectFields.size()) { @@ -47,55 +45,65 @@ public class SelectCorrector extends BaseSemanticCorrector { semanticParseInfo.getSqlInfo().setCorrectedS2SQL(querySql); } - protected String addFieldsToSelect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo, - String correctS2SQL) { + protected String addFieldsToSelect( + ChatQueryContext chatQueryContext, + SemanticParseInfo semanticParseInfo, + String correctS2SQL) { correctS2SQL = addTagDefaultFields(chatQueryContext, semanticParseInfo, correctS2SQL); Set selectFields = new HashSet<>(SqlSelectHelper.getSelectFields(correctS2SQL)); Set needAddFields = new HashSet<>(SqlSelectHelper.getGroupByFields(correctS2SQL)); - //decide whether add order by expression field to select + // decide whether add order by expression field to select Environment environment = ContextUtils.getBean(Environment.class); String correctorAdditionalInfo = environment.getProperty(ADDITIONAL_INFORMATION); - if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) { + if (StringUtils.isNotBlank(correctorAdditionalInfo) + && Boolean.parseBoolean(correctorAdditionalInfo)) { needAddFields.addAll(SqlSelectHelper.getOrderByFields(correctS2SQL)); } if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(needAddFields)) { return correctS2SQL; } needAddFields.removeAll(selectFields); - String addFieldsToSelectSql = SqlAddHelper.addFieldsToSelect(correctS2SQL, new ArrayList<>(needAddFields)); + String addFieldsToSelectSql = + SqlAddHelper.addFieldsToSelect(correctS2SQL, new ArrayList<>(needAddFields)); semanticParseInfo.getSqlInfo().setCorrectedS2SQL(addFieldsToSelectSql); return addFieldsToSelectSql; } - private String addTagDefaultFields(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo, - String correctS2SQL) { - //If it is in DETAIL mode and select *, add default metrics and dimensions. + private String addTagDefaultFields( + ChatQueryContext chatQueryContext, + SemanticParseInfo semanticParseInfo, + String correctS2SQL) { + // If it is in DETAIL mode and select *, add default metrics and dimensions. boolean hasAsterisk = SqlSelectFunctionHelper.hasAsterisk(correctS2SQL); if (!(hasAsterisk && QueryType.DETAIL.equals(semanticParseInfo.getQueryType()))) { return correctS2SQL; } Long dataSetId = semanticParseInfo.getDataSetId(); - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); Set needAddDefaultFields = new HashSet<>(); if (Objects.nonNull(dataSetSchema)) { if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultMetrics())) { - Set metrics = dataSetSchema.getTagDefaultMetrics() - .stream().map(schemaElement -> schemaElement.getName()) - .collect(Collectors.toSet()); + Set metrics = + dataSetSchema.getTagDefaultMetrics().stream() + .map(schemaElement -> schemaElement.getName()) + .collect(Collectors.toSet()); needAddDefaultFields.addAll(metrics); } if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultDimensions())) { - Set dimensions = dataSetSchema.getTagDefaultDimensions() - .stream().map(schemaElement -> schemaElement.getName()) - .collect(Collectors.toSet()); + Set dimensions = + dataSetSchema.getTagDefaultDimensions().stream() + .map(schemaElement -> schemaElement.getName()) + .collect(Collectors.toSet()); needAddDefaultFields.addAll(dimensions); } } // remove * in sql and add default fields. if (!CollectionUtils.isEmpty(needAddDefaultFields)) { - correctS2SQL = SqlRemoveHelper.removeAsteriskAndAddFields(correctS2SQL, needAddDefaultFields); + correctS2SQL = + SqlRemoveHelper.removeAsteriskAndAddFields(correctS2SQL, needAddDefaultFields); } return correctS2SQL; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SemanticCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SemanticCorrector.java index 2ddbb68a9..39ca2b5a2 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SemanticCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SemanticCorrector.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.chat.corrector; - import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; /** - * A semantic corrector checks validity of extracted semantic information and - * performs correction and optimization if needed. + * A semantic corrector checks validity of extracted semantic information and performs correction + * and optimization if needed. */ public interface SemanticCorrector { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java index 2538bb5d8..14ec04007 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.corrector; - import com.tencent.supersonic.common.jsqlparser.DateVisitor.DateBoundInfo; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlDateSelectHelper; @@ -20,9 +19,7 @@ import org.springframework.util.CollectionUtils; import java.util.List; import java.util.Objects; -/** - * Perform SQL corrections on the time in S2SQL. - */ +/** Perform SQL corrections on the time in S2SQL. */ @Slf4j public class TimeCorrector extends BaseSemanticCorrector { @@ -36,11 +33,13 @@ public class TimeCorrector extends BaseSemanticCorrector { addLowerBoundDate(semanticParseInfo); } - private void addDateIfNotExist(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + private void addDateIfNotExist( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); Long dataSetId = semanticParseInfo.getDataSetId(); - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension()) || Objects.isNull(dataSetSchema.getPartitionDimension().getName()) @@ -49,16 +48,22 @@ public class TimeCorrector extends BaseSemanticCorrector { } String partitionDimension = dataSetSchema.getPartitionDimension().getName(); if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(partitionDimension)) { - Pair startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, - semanticParseInfo.getQueryType()); + Pair startEndDate = + S2SqlDateHelper.getStartEndDate( + chatQueryContext, dataSetId, semanticParseInfo.getQueryType()); if (isValidDateRange(startEndDate)) { correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); String startDateLeft = startEndDate.getLeft(); String endDateRight = startEndDate.getRight(); - String condExpr = String.format(" ( %s >= '%s' and %s <= '%s' )", - partitionDimension, startDateLeft, partitionDimension, endDateRight); + String condExpr = + String.format( + " ( %s >= '%s' and %s <= '%s' )", + partitionDimension, + startDateLeft, + partitionDimension, + endDateRight); correctS2SQL = addConditionToSQL(correctS2SQL, condExpr); } } @@ -94,4 +99,4 @@ public class TimeCorrector extends BaseSemanticCorrector { return sql; } } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrector.java index bf645a961..02589d82a 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrector.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.corrector; - import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.headless.api.pojo.SchemaElement; @@ -9,11 +8,6 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.utils.QueryFilterParser; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -21,9 +15,13 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; -/** - * Perform SQL corrections on the "Where" section in S2SQL. - */ +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** Perform SQL corrections on the "Where" section in S2SQL. */ @Slf4j public class WhereCorrector extends BaseSemanticCorrector { @@ -33,7 +31,8 @@ public class WhereCorrector extends BaseSemanticCorrector { updateFieldValueByTechName(chatQueryContext, semanticParseInfo); } - protected void addQueryFilter(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + protected void addQueryFilter( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { String queryFilter = getQueryFilter(chatQueryContext.getQueryFilters()); String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); @@ -56,7 +55,8 @@ public class WhereCorrector extends BaseSemanticCorrector { return QueryFilterParser.parse(queryFilters); } - private void updateFieldValueByTechName(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + private void updateFieldValueByTechName( + ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); Long dataSetId = semanticParseInfo.getDataSetId(); List dimensions = semanticSchema.getDimensions(dataSetId); @@ -64,35 +64,61 @@ public class WhereCorrector extends BaseSemanticCorrector { if (CollectionUtils.isEmpty(dimensions)) { return; } - Map> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions); + Map> aliasAndBizNameToTechName = + getAliasAndBizNameToTechName(dimensions); String correctedS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); - String replaceSql = SqlReplaceHelper.replaceValue(correctedS2SQL, aliasAndBizNameToTechName); + String replaceSql = + SqlReplaceHelper.replaceValue(correctedS2SQL, aliasAndBizNameToTechName); semanticParseInfo.getSqlInfo().setCorrectedS2SQL(replaceSql); } - private Map> getAliasAndBizNameToTechName(List dimensions) { + private Map> getAliasAndBizNameToTechName( + List dimensions) { return dimensions.stream() - .filter(dimension -> Objects.nonNull(dimension) - && StringUtils.isNotEmpty(dimension.getName()) - && !CollectionUtils.isEmpty(dimension.getSchemaValueMaps())) - .collect(Collectors.toMap( - SchemaElement::getName, - dimension -> dimension.getSchemaValueMaps().stream() - .filter(valueMap -> Objects.nonNull(valueMap) - && StringUtils.isNotEmpty(valueMap.getTechName())) - .flatMap(valueMap -> { - Map map = new HashMap<>(); - if (StringUtils.isNotEmpty(valueMap.getBizName())) { - map.put(valueMap.getBizName(), valueMap.getTechName()); - } - if (!CollectionUtils.isEmpty(valueMap.getAlias())) { - valueMap.getAlias().stream() - .filter(StringUtils::isNotEmpty) - .forEach(alias -> map.put(alias, valueMap.getTechName())); - } - return map.entrySet().stream(); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - )); + .filter( + dimension -> + Objects.nonNull(dimension) + && StringUtils.isNotEmpty(dimension.getName()) + && !CollectionUtils.isEmpty(dimension.getSchemaValueMaps())) + .collect( + Collectors.toMap( + SchemaElement::getName, + dimension -> + dimension.getSchemaValueMaps().stream() + .filter( + valueMap -> + Objects.nonNull(valueMap) + && StringUtils.isNotEmpty( + valueMap + .getTechName())) + .flatMap( + valueMap -> { + Map map = + new HashMap<>(); + if (StringUtils.isNotEmpty( + valueMap.getBizName())) { + map.put( + valueMap.getBizName(), + valueMap.getTechName()); + } + if (!CollectionUtils.isEmpty( + valueMap.getAlias())) { + valueMap.getAlias().stream() + .filter( + StringUtils + ::isNotEmpty) + .forEach( + alias -> + map.put( + alias, + valueMap + .getTechName())); + } + return map.entrySet().stream(); + }) + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue)))); } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DataSetInfoStat.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DataSetInfoStat.java index a24c2ea69..4faa9cc23 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DataSetInfoStat.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DataSetInfoStat.java @@ -18,5 +18,4 @@ public class DataSetInfoStat implements Serializable { private long dimensionDataSetCount; private long dimensionValueDataSetCount; - -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DatabaseMapResult.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DatabaseMapResult.java index 60d87d5b1..ba5a0e9fd 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DatabaseMapResult.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DatabaseMapResult.java @@ -27,4 +27,4 @@ public class DatabaseMapResult extends MapResult { public int hashCode() { return Objects.hashCode(name, schemaElement); } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictUpdateMode.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictUpdateMode.java index cd6c74be3..85d2be561 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictUpdateMode.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictUpdateMode.java @@ -1,8 +1,6 @@ - package com.tencent.supersonic.headless.chat.knowledge; public enum DictUpdateMode { - OFFLINE_FULL("OFFLINE_FULL"), OFFLINE_MODEL("OFFLINE_MODEL"), REALTIME_ADD("REALTIME_ADD"), @@ -27,5 +25,4 @@ public enum DictUpdateMode { public String getValue() { return value; } - -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java index 4b9a7bd0e..127907e5b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.chat.knowledge; -import java.util.Objects; import lombok.Data; import lombok.ToString; -/*** - * word nature - */ +import java.util.Objects; + +/** * word nature */ @Data @ToString public class DictWord { @@ -24,7 +23,8 @@ public class DictWord { return false; } DictWord that = (DictWord) o; - return Objects.equals(word, that.word) && Objects.equals(natureWithFrequency, that.natureWithFrequency); + return Objects.equals(word, that.word) + && Objects.equals(natureWithFrequency, that.natureWithFrequency); } @Override diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictionaryAttributeUtil.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictionaryAttributeUtil.java index 8ce079d15..dcae4b7f0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictionaryAttributeUtil.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictionaryAttributeUtil.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.chat.knowledge; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; + import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -12,38 +13,52 @@ import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.IntStream; -/** - * Dictionary Attribute Util - */ +/** Dictionary Attribute Util */ public class DictionaryAttributeUtil { - public static CoreDictionary.Attribute getAttribute(CoreDictionary.Attribute old, CoreDictionary.Attribute add) { + public static CoreDictionary.Attribute getAttribute( + CoreDictionary.Attribute old, CoreDictionary.Attribute add) { Map map = new HashMap<>(); Map originalMap = new HashMap<>(); - IntStream.range(0, old.nature.length).boxed().forEach(i -> { - map.put(old.nature[i], old.frequency[i]); - if (Objects.nonNull(old.originals)) { - originalMap.put(old.nature[i], old.originals[i]); - } - }); - IntStream.range(0, add.nature.length).boxed().forEach(i -> { - map.put(add.nature[i], add.frequency[i]); - if (Objects.nonNull(add.originals)) { - originalMap.put(add.nature[i], add.originals[i]); - } - }); - List> list = new LinkedList>(map.entrySet()); - Collections.sort(list, new Comparator>() { - public int compare(Map.Entry o1, Map.Entry o2) { - return o2.getValue() - o1.getValue(); - } - }); - String[] originals = list.stream().map(l -> originalMap.get(l.getKey())).toArray(String[]::new); - CoreDictionary.Attribute attribute = new CoreDictionary.Attribute( - list.stream().map(i -> i.getKey()).collect(Collectors.toList()).toArray(new Nature[0]), - list.stream().map(i -> i.getValue()).mapToInt(Integer::intValue).toArray(), - originals, - list.stream().map(i -> i.getValue()).findFirst().get()); + IntStream.range(0, old.nature.length) + .boxed() + .forEach( + i -> { + map.put(old.nature[i], old.frequency[i]); + if (Objects.nonNull(old.originals)) { + originalMap.put(old.nature[i], old.originals[i]); + } + }); + IntStream.range(0, add.nature.length) + .boxed() + .forEach( + i -> { + map.put(add.nature[i], add.frequency[i]); + if (Objects.nonNull(add.originals)) { + originalMap.put(add.nature[i], add.originals[i]); + } + }); + List> list = + new LinkedList>(map.entrySet()); + Collections.sort( + list, + new Comparator>() { + public int compare( + Map.Entry o1, Map.Entry o2) { + return o2.getValue() - o1.getValue(); + } + }); + String[] originals = + list.stream().map(l -> originalMap.get(l.getKey())).toArray(String[]::new); + CoreDictionary.Attribute attribute = + new CoreDictionary.Attribute( + list.stream() + .map(i -> i.getKey()) + .collect(Collectors.toList()) + .toArray(new Nature[0]), + list.stream().map(i -> i.getValue()).mapToInt(Integer::intValue).toArray(), + originals, + list.stream().map(i -> i.getValue()).findFirst().get()); return attribute; } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/EmbeddingResult.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/EmbeddingResult.java index 04486b6b0..1eebf3d0c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/EmbeddingResult.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/EmbeddingResult.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.chat.knowledge; import com.google.common.base.Objects; -import java.util.Map; import lombok.Data; import lombok.ToString; +import java.util.Map; + @Data @ToString public class EmbeddingResult extends MapResult { @@ -31,4 +32,4 @@ public class EmbeddingResult extends MapResult { public int hashCode() { return Objects.hashCode(id); } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HadoopFileIOAdapter.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HadoopFileIOAdapter.java index 3b24e16eb..5ed01798a 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HadoopFileIOAdapter.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HadoopFileIOAdapter.java @@ -1,16 +1,16 @@ package com.tencent.supersonic.headless.chat.knowledge; import com.hankcs.hanlp.corpus.io.IIOAdapter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; - import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; + @Slf4j public class HadoopFileIOAdapter implements IIOAdapter { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HanlpMapResult.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HanlpMapResult.java index 301f95d34..767bcd141 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HanlpMapResult.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/HanlpMapResult.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.chat.knowledge; import com.google.common.base.Objects; -import java.util.List; import lombok.Data; import lombok.ToString; +import java.util.List; + @Data @ToString public class HanlpMapResult extends MapResult { @@ -29,7 +30,8 @@ public class HanlpMapResult extends MapResult { return false; } HanlpMapResult hanlpMapResult = (HanlpMapResult) o; - return Objects.equal(name, hanlpMapResult.name) && Objects.equal(natures, hanlpMapResult.natures); + return Objects.equal(name, hanlpMapResult.name) + && Objects.equal(natures, hanlpMapResult.natures); } @Override @@ -40,5 +42,4 @@ public class HanlpMapResult extends MapResult { public void setOffset(int offset) { this.offset = offset; } - -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java index 6bf2ecc98..de3cc9423 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java @@ -17,17 +17,25 @@ public class KnowledgeBaseService { public void updateSemanticKnowledge(List natures) { - List prefixes = natures.stream() - .filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType())) - .collect(Collectors.toList()); + List prefixes = + natures.stream() + .filter( + entry -> + !entry.getNatureWithFrequency() + .contains(DictWordType.SUFFIX.getType())) + .collect(Collectors.toList()); for (DictWord nature : prefixes) { HanlpHelper.addToCustomDictionary(nature); } - List suffixes = natures.stream() - .filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType())) - .collect(Collectors.toList()); + List suffixes = + natures.stream() + .filter( + entry -> + entry.getNatureWithFrequency() + .contains(DictWordType.SUFFIX.getType())) + .collect(Collectors.toList()); SearchService.loadSuffix(suffixes); } @@ -56,26 +64,35 @@ public class KnowledgeBaseService { return HanlpHelper.getTerms(text, modelIdToDataSetIds); } - public List prefixSearch(String key, int limit, - Map> modelIdToDataSetIds, - Set detectDataSetIds) { + public List prefixSearch( + String key, + int limit, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds); } - public List prefixSearchByModel(String key, int limit, - Map> modelIdToDataSetIds, - Set detectDataSetIds) { + public List prefixSearchByModel( + String key, + int limit, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds); } - public List suffixSearch(String key, int limit, Map> modelIdToDataSetIds, Set detectDataSetIds) { + public List suffixSearch( + String key, + int limit, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds); } - public List suffixSearchByModel(String key, int limit, Map> modelIdToDataSetIds, Set detectDataSetIds) { + public List suffixSearchByModel( + String key, + int limit, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds); } - -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java index 6c4fb6e72..ee7213e97 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java @@ -1,13 +1,14 @@ package com.tencent.supersonic.headless.chat.knowledge; -import java.io.Serializable; import lombok.Data; import lombok.ToString; +import java.io.Serializable; + @Data @ToString public class MapResult implements Serializable { protected String name; protected String detectWord; -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java index 9933998f2..fe98b7637 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java @@ -26,27 +26,30 @@ import java.util.stream.Stream; @Slf4j public class MetaEmbeddingService { - @Autowired - private EmbeddingService embeddingService; - @Autowired - private EmbeddingConfig embeddingConfig; + @Autowired private EmbeddingService embeddingService; + @Autowired private EmbeddingConfig embeddingConfig; - public List retrieveQuery(RetrieveQuery retrieveQuery, int num, - Map> modelIdToDataSetIds, - Set detectDataSetIds) { + public List retrieveQuery( + RetrieveQuery retrieveQuery, + int num, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { // dataSetIds->modelIds Set allModels = NatureHelper.getModelIds(modelIdToDataSetIds, detectDataSetIds); if (CollectionUtils.isNotEmpty(allModels)) { Map filterCondition = new HashMap<>(); - filterCondition.put("modelId", allModels.stream() - .map(modelId -> modelId + DictWordType.NATURE_SPILT) - .collect(Collectors.toList())); + filterCondition.put( + "modelId", + allModels.stream() + .map(modelId -> modelId + DictWordType.NATURE_SPILT) + .collect(Collectors.toList())); retrieveQuery.setFilterCondition(filterCondition); } String collectionName = embeddingConfig.getMetaCollectionName(); - List resultList = embeddingService.retrieveQuery(collectionName, retrieveQuery, num); + List resultList = + embeddingService.retrieveQuery(collectionName, retrieveQuery, num); if (CollectionUtils.isEmpty(resultList)) { return new ArrayList<>(); } @@ -57,31 +60,44 @@ public class MetaEmbeddingService { .collect(Collectors.toList()); } - private static RetrieveQueryResult getRetrieveQueryResult(Map> modelIdToDataSetIds, RetrieveQueryResult result) { + private static RetrieveQueryResult getRetrieveQueryResult( + Map> modelIdToDataSetIds, RetrieveQueryResult result) { List retrievals = result.getRetrieval(); if (CollectionUtils.isEmpty(retrievals)) { return result; } // Process each Retrieval object. - List updatedRetrievals = retrievals.stream() - .flatMap(retrieval -> { - Long modelId = Retrieval.getLongId(retrieval.getMetadata().get("modelId")); - List dataSetIds = modelIdToDataSetIds.get(modelId); + List updatedRetrievals = + retrievals.stream() + .flatMap( + retrieval -> { + Long modelId = + Retrieval.getLongId( + retrieval.getMetadata().get("modelId")); + List dataSetIds = modelIdToDataSetIds.get(modelId); - if (CollectionUtils.isEmpty(dataSetIds)) { - return Stream.of(retrieval); - } + if (CollectionUtils.isEmpty(dataSetIds)) { + return Stream.of(retrieval); + } - return dataSetIds.stream().map(dataSetId -> { - Retrieval newRetrieval = new Retrieval(); - BeanUtils.copyProperties(retrieval, newRetrieval); - newRetrieval.getMetadata().putIfAbsent("dataSetId", dataSetId + Constants.UNDERLINE); - return newRetrieval; - }); - }) - .collect(Collectors.toList()); + return dataSetIds.stream() + .map( + dataSetId -> { + Retrieval newRetrieval = new Retrieval(); + BeanUtils.copyProperties( + retrieval, newRetrieval); + newRetrieval + .getMetadata() + .putIfAbsent( + "dataSetId", + dataSetId + + Constants + .UNDERLINE); + return newRetrieval; + }); + }) + .collect(Collectors.toList()); result.setRetrieval(updatedRetrievals); return result; } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java index 6b9fd50d7..bb31c09f2 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.headless.chat.knowledge; -import static com.hankcs.hanlp.utility.Predefine.logger; - import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.collection.trie.DoubleArrayTrie; import com.hankcs.hanlp.collection.trie.bintrie.BinTrie; @@ -34,11 +32,14 @@ import java.util.PriorityQueue; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import static com.hankcs.hanlp.utility.Predefine.logger; + public class MultiCustomDictionary extends DynamicCustomDictionary { public static int MAX_SIZE = 10; public static Boolean removeDuplicates = true; - public static ConcurrentHashMap> NATURE_TO_VALUES = new ConcurrentHashMap<>(); + public static ConcurrentHashMap> NATURE_TO_VALUES = + new ConcurrentHashMap<>(); private static boolean addToSuggesterTrie = true; public MultiCustomDictionary() { @@ -49,8 +50,9 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { super(path); } - /*** - * load dictionary + /** + * * load dictionary + * * @param path * @param defaultNature * @param map @@ -58,15 +60,20 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { * @param addToSuggeterTrie * @return */ - public static boolean load(String path, Nature defaultNature, TreeMap map, - LinkedHashSet customNatureCollector, boolean addToSuggeterTrie) { + public static boolean load( + String path, + Nature defaultNature, + TreeMap map, + LinkedHashSet customNatureCollector, + boolean addToSuggeterTrie) { try { String splitter = "\\s"; if (path.endsWith(".csv")) { splitter = ","; } - BufferedReader br = new BufferedReader(new InputStreamReader(IOUtil.newInputStream(path), "UTF-8")); + BufferedReader br = + new BufferedReader(new InputStreamReader(IOUtil.newInputStream(path), "UTF-8")); boolean firstLine = true; while (true) { @@ -105,14 +112,15 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { attribute = new CoreDictionary.Attribute(natureCount); for (int i = 0; i < natureCount; ++i) { - attribute.nature[i] = LexiconUtility.convertStringToNature(param[1 + 2 * i], - customNatureCollector); + attribute.nature[i] = + LexiconUtility.convertStringToNature( + param[1 + 2 * i], customNatureCollector); attribute.frequency[i] = Integer.parseInt(param[2 + 2 * i]); attribute.originals[i] = original; attribute.totalFrequency += attribute.frequency[i]; } } - //attribute.original = original; + // attribute.original = original; if (removeDuplicates && map.containsKey(word)) { attribute = DictionaryAttributeUtil.getAttribute(map.get(word), attribute); @@ -125,8 +133,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { Nature nature = attribute.nature[i]; PriorityQueue priorityQueue = NATURE_TO_VALUES.get(nature.toString()); if (Objects.isNull(priorityQueue)) { - priorityQueue = new PriorityQueue<>(MAX_SIZE, - Comparator.comparingInt(Term::getFrequency).reversed()); + priorityQueue = + new PriorityQueue<>( + MAX_SIZE, + Comparator.comparingInt(Term::getFrequency).reversed()); NATURE_TO_VALUES.put(nature.toString(), priorityQueue); } Term term = new Term(word, nature); @@ -150,14 +160,19 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { return false; } else { logger.info( - "自定义词典加载成功:" + this.dat.size() + "个词条，耗时" + (System.currentTimeMillis() - start) + "ms"); + "自定义词典加载成功:" + + this.dat.size() + + "个词条，耗时" + + (System.currentTimeMillis() - start) + + "ms"); this.path = path; return true; } } - /*** - * load main dictionary + /** + * * load main dictionary + * * @param mainPath * @param path * @param dat @@ -165,9 +180,12 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { * @param addToSuggestTrie * @return */ - public static boolean loadMainDictionary(String mainPath, String[] path, - DoubleArrayTrie dat, boolean isCache, - boolean addToSuggestTrie) { + public static boolean loadMainDictionary( + String mainPath, + String[] path, + DoubleArrayTrie dat, + boolean isCache, + boolean addToSuggestTrie) { logger.info("自定义词典开始加载:" + mainPath); if (loadDat(mainPath, dat)) { return true; @@ -186,7 +204,9 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { p = file.getParent() + File.separator + fileName.substring(0, cut); try { - defaultNature = LexiconUtility.convertStringToNature(nature, customNatureCollector); + defaultNature = + LexiconUtility.convertStringToNature( + nature, customNatureCollector); } catch (Exception var16) { logger.severe("配置文件【" + p + "】写错了！" + var16); continue; @@ -194,7 +214,8 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { } logger.info("以默认词性[" + defaultNature + "]加载自定义词典" + p + "中……"); - boolean success = load(p, defaultNature, map, customNatureCollector, addToSuggestTrie); + boolean success = + load(p, defaultNature, map, customNatureCollector, addToSuggestTrie); if (!success) { logger.warning("失败：" + p); } @@ -214,13 +235,16 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { // 缓存成dat文件，下次加载会快很多 logger.info("正在缓存词典为dat文件……"); // 缓存值文件 - List attributeList = new LinkedList(); + List attributeList = + new LinkedList(); for (Map.Entry entry : map.entrySet()) { attributeList.add(entry.getValue()); } - DataOutputStream out = new DataOutputStream( - new BufferedOutputStream(IOUtil.newOutputStream(mainPath + ".bin"))); + DataOutputStream out = + new DataOutputStream( + new BufferedOutputStream( + IOUtil.newOutputStream(mainPath + ".bin"))); if (customNatureCollector.isEmpty()) { for (int i = Nature.begin.ordinal() + 1; i < Nature.values().length; ++i) { Nature nature = Nature.values()[i]; @@ -247,7 +271,8 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { logger.severe("自定义词典" + mainPath + "读取错误！" + var18); return false; } catch (Exception var19) { - logger.warning("自定义词典" + mainPath + "缓存失败！\n" + TextUtility.exceptionToString(var19)); + logger.warning( + "自定义词典" + mainPath + "缓存失败！\n" + TextUtility.exceptionToString(var19)); } return true; @@ -262,7 +287,8 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { return loadDat(path, HanLP.Config.CustomDictionaryPath, dat); } - public static boolean loadDat(String path, String[] customDicPath, DoubleArrayTrie dat) { + public static boolean loadDat( + String path, String[] customDicPath, DoubleArrayTrie dat) { try { if (HanLP.Config.CustomDictionaryAutoRefreshCache && DynamicCustomDictionary.isDicNeedUpdate(path, customDicPath)) { @@ -348,11 +374,11 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { IOUtil.deleteFile(this.path[0] + ".bin"); Boolean loadCacheOk = this.loadDat(this.path[0], this.path, this.dat); if (!loadCacheOk) { - return this.loadMainDictionary(this.path[0], this.path, this.dat, true, addToSuggesterTrie); + return this.loadMainDictionary( + this.path[0], this.path, this.dat, true, addToSuggesterTrie); } } return false; - } public synchronized boolean insert(String word, String natureWithFrequency) { @@ -362,8 +388,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { if (HanLP.Config.Normalization) { word = CharTable.convert(word); } - CoreDictionary.Attribute att = natureWithFrequency == null ? new CoreDictionary.Attribute(Nature.nz, 1) - : CoreDictionary.Attribute.create(natureWithFrequency); + CoreDictionary.Attribute att = + natureWithFrequency == null + ? new CoreDictionary.Attribute(Nature.nz, 1) + : CoreDictionary.Attribute.create(natureWithFrequency); boolean isLetters = isLetters(word); word = getWordBySpace(word); String original = null; @@ -382,7 +410,7 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { if (this.trie == null) { this.trie = new BinTrie(); } - //att.original = original; + // att.original = original; att.setOriginals(original); if (this.trie.containsKey(word)) { att = DictionaryAttributeUtil.getAttribute(this.trie.get(word), att); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java index 0c1d64101..c72dde3b5 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java @@ -36,85 +36,130 @@ public class SearchService { suffixTrie = new BinTrie<>(); } - /*** - * prefix Search + /** + * * prefix Search + * * @param key * @return */ - public static List prefixSearch(String key, int limit, Map> modelIdToDataSetIds, + public static List prefixSearch( + String key, + int limit, + Map> modelIdToDataSetIds, Set detectDataSetIds) { return prefixSearch(key, limit, trie, modelIdToDataSetIds, detectDataSetIds); } - public static List prefixSearch(String key, int limit, BinTrie> binTrie, - Map> modelIdToDataSetIds, Set detectDataSetIds) { + public static List prefixSearch( + String key, + int limit, + BinTrie> binTrie, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { Set>> result = search(key, binTrie); - List hanlpMapResults = result.stream().map( - entry -> { - String name = entry.getKey().replace("#", " "); - return new HanlpMapResult(name, entry.getValue(), key); - } - ).sorted((a, b) -> -(b.getName().length() - a.getName().length())) - .collect(Collectors.toList()); - hanlpMapResults = transformAndFilterByDataSet(hanlpMapResults, modelIdToDataSetIds, - detectDataSetIds, limit); + List hanlpMapResults = + result.stream() + .map( + entry -> { + String name = entry.getKey().replace("#", " "); + return new HanlpMapResult(name, entry.getValue(), key); + }) + .sorted((a, b) -> -(b.getName().length() - a.getName().length())) + .collect(Collectors.toList()); + hanlpMapResults = + transformAndFilterByDataSet( + hanlpMapResults, modelIdToDataSetIds, detectDataSetIds, limit); return hanlpMapResults; } - /*** - * suffix Search + /** + * * suffix Search + * * @param key * @return */ - public static List suffixSearch(String key, int limit, Map> modelIdToDataSetIds, + public static List suffixSearch( + String key, + int limit, + Map> modelIdToDataSetIds, Set detectDataSetIds) { String reverseDetectSegment = StringUtils.reverse(key); - return suffixSearch(reverseDetectSegment, limit, suffixTrie, modelIdToDataSetIds, detectDataSetIds); + return suffixSearch( + reverseDetectSegment, limit, suffixTrie, modelIdToDataSetIds, detectDataSetIds); } - public static List suffixSearch(String key, int limit, BinTrie> binTrie, - Map> modelIdToDataSetIds, Set detectDataSetIds) { + public static List suffixSearch( + String key, + int limit, + BinTrie> binTrie, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { Set>> result = search(key, binTrie); - List hanlpMapResults = result.stream().map( - entry -> { - String name = entry.getKey().replace("#", " "); - List natures = entry.getValue().stream() - .map(nature -> nature.replaceAll(DictWordType.SUFFIX.getType(), "")) - .collect(Collectors.toList()); - name = StringUtils.reverse(name); - return new HanlpMapResult(name, natures, key); - } - ).sorted((a, b) -> -(b.getName().length() - a.getName().length())) + List hanlpMapResults = + result.stream() + .map( + entry -> { + String name = entry.getKey().replace("#", " "); + List natures = + entry.getValue().stream() + .map( + nature -> + nature.replaceAll( + DictWordType.SUFFIX + .getType(), + "")) + .collect(Collectors.toList()); + name = StringUtils.reverse(name); + return new HanlpMapResult(name, natures, key); + }) + .sorted((a, b) -> -(b.getName().length() - a.getName().length())) + .collect(Collectors.toList()); + return transformAndFilterByDataSet( + hanlpMapResults, modelIdToDataSetIds, detectDataSetIds, limit); + } + + private static List transformAndFilterByDataSet( + List hanlpMapResults, + Map> modelIdToDataSetIds, + Set detectDataSetIds, + int limit) { + return hanlpMapResults.stream() + .peek( + hanlpMapResult -> { + List natures = + hanlpMapResult.getNatures().stream() + .map( + nature -> + NatureHelper.changeModel2DataSet( + nature, modelIdToDataSetIds)) + .flatMap(Collection::stream) + .filter( + nature -> { + if (CollectionUtils.isEmpty( + detectDataSetIds)) { + return true; + } + Long dataSetId = + NatureHelper.getDataSetId(nature); + if (dataSetId != null) { + return detectDataSetIds.contains( + dataSetId); + } + return false; + }) + .collect(Collectors.toList()); + hanlpMapResult.setNatures(natures); + }) + .filter(hanlpMapResult -> !CollectionUtils.isEmpty(hanlpMapResult.getNatures())) + .limit(limit) .collect(Collectors.toList()); - return transformAndFilterByDataSet(hanlpMapResults, modelIdToDataSetIds, detectDataSetIds, limit); } - private static List transformAndFilterByDataSet(List hanlpMapResults, - Map> modelIdToDataSetIds, - Set detectDataSetIds, int limit) { - return hanlpMapResults.stream().peek(hanlpMapResult -> { - List natures = hanlpMapResult.getNatures().stream() - .map(nature -> NatureHelper.changeModel2DataSet(nature, modelIdToDataSetIds)) - .flatMap(Collection::stream) - .filter(nature -> { - if (CollectionUtils.isEmpty(detectDataSetIds)) { - return true; - } - Long dataSetId = NatureHelper.getDataSetId(nature); - if (dataSetId != null) { - return detectDataSetIds.contains(dataSetId); - } - return false; - }).collect(Collectors.toList()); - hanlpMapResult.setNatures(natures); - }).filter(hanlpMapResult -> !CollectionUtils.isEmpty(hanlpMapResult.getNatures())) - .limit(limit).collect(Collectors.toList()); - } - - private static Set>> search(String key, - BinTrie> binTrie) { + private static Set>> search( + String key, BinTrie> binTrie) { key = key.toLowerCase(); - Set>> entrySet = new TreeSet>>(); + Set>> entrySet = + new TreeSet>>(); StringBuilder sb = new StringBuilder(); if (StringUtils.isNotBlank(key)) { @@ -152,11 +197,14 @@ public class SearchService { } TreeMap map = new TreeMap(); for (DictWord suffix : suffixes) { - CoreDictionary.Attribute attributeNew = suffix.getNatureWithFrequency() == null - ? new CoreDictionary.Attribute(Nature.nz, 1) - : CoreDictionary.Attribute.create(suffix.getNatureWithFrequency()); + CoreDictionary.Attribute attributeNew = + suffix.getNatureWithFrequency() == null + ? new CoreDictionary.Attribute(Nature.nz, 1) + : CoreDictionary.Attribute.create(suffix.getNatureWithFrequency()); if (map.containsKey(suffix.getWord())) { - attributeNew = DictionaryAttributeUtil.getAttribute(map.get(suffix.getWord()), attributeNew); + attributeNew = + DictionaryAttributeUtil.getAttribute( + map.get(suffix.getWord()), attributeNew); } map.put(suffix.getWord(), attributeNew); } @@ -179,15 +227,18 @@ public class SearchService { if (Objects.nonNull(natures) && natures.length > 0) { trie.put(dictWord.getWord(), getValue(natures)); } - if (dictWord.getNature().contains(DictWordType.METRIC.getType()) || dictWord.getNature() - .contains(DictWordType.DIMENSION.getType())) { + if (dictWord.getNature().contains(DictWordType.METRIC.getType()) + || dictWord.getNature().contains(DictWordType.DIMENSION.getType())) { suffixTrie.remove(dictWord.getWord()); } } public static List getDimensionValue(DimensionValueReq dimensionValueReq) { - String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT - + dimensionValueReq.getElementID(); + String nature = + DictWordType.NATURE_SPILT + + dimensionValueReq.getModelId() + + DictWordType.NATURE_SPILT + + dimensionValueReq.getElementID(); PriorityQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); if (CollectionUtils.isEmpty(terms)) { return new ArrayList<>(); @@ -195,4 +246,3 @@ public class SearchService { return terms.stream().map(term -> term.getWord()).collect(Collectors.toList()); } } - diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordBuilder.java index 517c0621f..3f931d5ac 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordBuilder.java @@ -2,14 +2,12 @@ package com.tencent.supersonic.headless.chat.knowledge.builder; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; +import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.List; -import lombok.extern.slf4j.Slf4j; -/** - * base word nature - */ +/** base word nature */ @Slf4j public abstract class BaseWordBuilder { @@ -36,5 +34,4 @@ public abstract class BaseWordBuilder { } protected abstract List doGet(String word, SchemaElement schemaElement); - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordWithAliasBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordWithAliasBuilder.java index c53ee7576..67c2b7066 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordWithAliasBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/BaseWordWithAliasBuilder.java @@ -2,14 +2,15 @@ package com.tencent.supersonic.headless.chat.knowledge.builder; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; -import org.springframework.util.CollectionUtils; public abstract class BaseWordWithAliasBuilder extends BaseWordBuilder { - public abstract DictWord getOneWordNature(String word, SchemaElement schemaElement, boolean isSuffix); + public abstract DictWord getOneWordNature( + String word, SchemaElement schemaElement, boolean isSuffix); public List getOneWordNatureAlias(SchemaElement schemaElement, boolean isSuffix) { List dictWords = new ArrayList<>(); @@ -22,5 +23,4 @@ public abstract class BaseWordWithAliasBuilder extends BaseWordBuilder { } return dictWords; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/DimensionWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/DimensionWordBuilder.java index 56d10d6b2..598bb3dfe 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/DimensionWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/DimensionWordBuilder.java @@ -4,14 +4,12 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; - -import java.util.List; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -/** - * dimension word nature - */ +import java.util.List; + +/** dimension word nature */ @Service public class DimensionWordBuilder extends BaseWordWithAliasBuilder { @@ -31,14 +29,22 @@ public class DimensionWordBuilder extends BaseWordWithAliasBuilder { DictWord dictWord = new DictWord(); dictWord.setWord(word); Long modelId = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() - + DictWordType.DIMENSION.getType(); + String nature = + DictWordType.NATURE_SPILT + + modelId + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.DIMENSION.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() - + DictWordType.SUFFIX.getType() + DictWordType.DIMENSION.getType(); + nature = + DictWordType.NATURE_SPILT + + modelId + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.SUFFIX.getType() + + DictWordType.DIMENSION.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); return dictWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/EntityWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/EntityWordBuilder.java index 85003d3d3..bde4f6c64 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/EntityWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/EntityWordBuilder.java @@ -1,15 +1,14 @@ package com.tencent.supersonic.headless.chat.knowledge.builder; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -24,17 +23,19 @@ public class EntityWordBuilder extends BaseWordWithAliasBuilder { result.add(getOneWordNature(word, schemaElement, false)); result.addAll(getOneWordNatureAlias(schemaElement, false)); return result; - } @Override public DictWord getOneWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { - String nature = DictWordType.NATURE_SPILT + schemaElement.getModel() - + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getType(); + String nature = + DictWordType.NATURE_SPILT + + schemaElement.getModel() + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.ENTITY.getType(); DictWord dictWord = new DictWord(); dictWord.setWord(word); dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY * 2, nature)); return dictWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/MetricWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/MetricWordBuilder.java index fbfc1cd34..002238a96 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/MetricWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/MetricWordBuilder.java @@ -4,14 +4,12 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; - -import java.util.List; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -/** - * Metric DictWord - */ +import java.util.List; + +/** Metric DictWord */ @Service public class MetricWordBuilder extends BaseWordWithAliasBuilder { @@ -31,14 +29,22 @@ public class MetricWordBuilder extends BaseWordWithAliasBuilder { DictWord dictWord = new DictWord(); dictWord.setWord(word); Long modelId = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() - + DictWordType.METRIC.getType(); + String nature = + DictWordType.NATURE_SPILT + + modelId + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.METRIC.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() - + DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType(); + nature = + DictWordType.NATURE_SPILT + + modelId + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.SUFFIX.getType() + + DictWordType.METRIC.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); return dictWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ModelWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ModelWordBuilder.java index 1885abd5f..474de5473 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ModelWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ModelWordBuilder.java @@ -4,15 +4,13 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; - -import java.util.List; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -/** - * model word nature - */ +import java.util.List; +import java.util.Objects; + +/** model word nature */ @Service @Slf4j public class ModelWordBuilder extends BaseWordWithAliasBuilder { @@ -35,5 +33,4 @@ public class ModelWordBuilder extends BaseWordWithAliasBuilder { dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); return dictWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/TermWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/TermWordBuilder.java index 8e27a80de..adc7ce18d 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/TermWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/TermWordBuilder.java @@ -9,9 +9,7 @@ import org.springframework.stereotype.Service; import java.util.List; -/** - * Metric DictWord - */ +/** Metric DictWord */ @Service public class TermWordBuilder extends BaseWordWithAliasBuilder { @@ -31,14 +29,22 @@ public class TermWordBuilder extends BaseWordWithAliasBuilder { DictWord dictWord = new DictWord(); dictWord.setWord(word); Long dataSet = schemaElement.getDataSetId(); - String nature = DictWordType.NATURE_SPILT + dataSet + DictWordType.NATURE_SPILT + schemaElement.getId() - + DictWordType.TERM.getType(); + String nature = + DictWordType.NATURE_SPILT + + dataSet + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.TERM.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + dataSet + DictWordType.NATURE_SPILT + schemaElement.getId() - + DictWordType.SUFFIX.getType() + DictWordType.TERM.getType(); + nature = + DictWordType.NATURE_SPILT + + dataSet + + DictWordType.NATURE_SPILT + + schemaElement.getId() + + DictWordType.SUFFIX.getType() + + DictWordType.TERM.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); return dictWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ValueWordBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ValueWordBuilder.java index 983f72e8c..f316589d6 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ValueWordBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/ValueWordBuilder.java @@ -1,15 +1,14 @@ package com.tencent.supersonic.headless.chat.knowledge.builder; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DictWord; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -27,10 +26,13 @@ public class ValueWordBuilder extends BaseWordWithAliasBuilder { public DictWord getOneWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); Long modelId = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId(); + String nature = + DictWordType.NATURE_SPILT + + modelId + + DictWordType.NATURE_SPILT + + schemaElement.getId(); dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); dictWord.setWord(word); return dictWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/WordBuilderFactory.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/WordBuilderFactory.java index 7806463a3..ef60a2314 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/WordBuilderFactory.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/builder/WordBuilderFactory.java @@ -1,13 +1,11 @@ package com.tencent.supersonic.headless.chat.knowledge.builder; - import com.tencent.supersonic.common.pojo.enums.DictWordType; + import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -/** - * DictWord Strategy Factory - */ +/** DictWord Strategy Factory */ public class WordBuilderFactory { private static Map wordNatures = new ConcurrentHashMap<>(); @@ -24,4 +22,4 @@ public class WordBuilderFactory { public static BaseWordBuilder get(DictWordType strategyType) { return wordNatures.get(strategyType); } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/ChatLocalFileConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/ChatLocalFileConfig.java index c04afd6ea..227c77878 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/ChatLocalFileConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/ChatLocalFileConfig.java @@ -8,13 +8,11 @@ import org.springframework.context.annotation.Configuration; import java.io.FileNotFoundException; - @Data @Configuration @Slf4j public class ChatLocalFileConfig { - @Value("${s2.dict.directory.latest:/data/dictionary/custom}") private String dictDirectoryLatest; @@ -38,4 +36,4 @@ public class ChatLocalFileConfig { } return hanlpPropertiesPath; } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java index fa0cd70b0..42dbe87af 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java @@ -9,8 +9,7 @@ import java.util.List; public interface FileHandler { /** - * backup files to a specific directory - * config: dict.directory.backup + * backup files to a specific directory config: dict.directory.backup * * @param fileName */ @@ -26,8 +25,7 @@ public interface FileHandler { Boolean existPath(String path); /** - * write data to a specific file, - * config dir: dict.directory.latest + * write data to a specific file, config dir: dict.directory.latest * * @param data * @param fileName @@ -43,8 +41,7 @@ public interface FileHandler { String getDictRootPath(); /** - * delete dictionary file - * automatic backup + * delete dictionary file automatic backup * * @param fileName * @return diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java index c61169651..2172d4452 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java @@ -8,16 +8,15 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.io.BufferedWriter; import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -52,7 +51,6 @@ public class FileHandlerImpl implements FileHandler { } catch (IOException e) { log.info("Failed to copy file: " + e.getMessage()); } - } @Override @@ -83,8 +81,12 @@ public class FileHandlerImpl implements FileHandler { String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; Long fileLineNum = getFileLineNum(filePath); Integer startLine = (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1; - Integer endLine = Integer.valueOf( - Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + ""); + Integer endLine = + Integer.valueOf( + Math.min( + dictValueReq.getCurrent() * dictValueReq.getPageSize(), + fileLineNum) + + ""); List dictValueRespList = getFileData(filePath, startLine, endLine); dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); @@ -110,17 +112,16 @@ public class FileHandlerImpl implements FileHandler { List fileData = new ArrayList<>(); try (Stream lines = Files.lines(Paths.get(filePath))) { - fileData = lines - .skip(startLine - 1) - .limit(endLine - startLine + 1) - .map(lineStr -> convert2Resp(lineStr)) - .filter(line -> Objects.nonNull(line)) - .collect(Collectors.toList()); + fileData = + lines.skip(startLine - 1) + .limit(endLine - startLine + 1) + .map(lineStr -> convert2Resp(lineStr)) + .filter(line -> Objects.nonNull(line)) + .collect(Collectors.toList()); } catch (IOException e) { log.warn("[getFileData] e:{}", e); } return fileData; - } private DictValueResp convert2Resp(String lineStr) { @@ -138,8 +139,7 @@ public class FileHandlerImpl implements FileHandler { private Long getFileLineNum(String filePath) { try (Stream lines = Files.lines(Paths.get(filePath))) { - Long lineCount = lines - .count(); + Long lineCount = lines.count(); return lineCount; } catch (IOException e) { e.printStackTrace(); @@ -204,8 +204,9 @@ public class FileHandlerImpl implements FileHandler { private BufferedWriter getWriter(String filePath, Boolean append) throws IOException { if (append) { - return Files.newBufferedWriter(Paths.get(filePath), StandardCharsets.UTF_8, StandardOpenOption.APPEND); + return Files.newBufferedWriter( + Paths.get(filePath), StandardCharsets.UTF_8, StandardOpenOption.APPEND); } return Files.newBufferedWriter(Paths.get(filePath), StandardCharsets.UTF_8); } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/LocalFileConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/LocalFileConfig.java index 4415c77c3..8b646912f 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/LocalFileConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/LocalFileConfig.java @@ -13,7 +13,6 @@ import java.io.FileNotFoundException; @Slf4j public class LocalFileConfig { - @Value("${s2.dict.directory.latest:/data/dictionary/custom}") private String dictDirectoryLatest; @@ -36,5 +35,4 @@ public class LocalFileConfig { } return ""; } - -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java index eae588614..3b815543b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java @@ -2,14 +2,13 @@ package com.tencent.supersonic.headless.chat.knowledge.helper; import com.hankcs.hanlp.HanLP.Config; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; +import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import lombok.extern.slf4j.Slf4j; - @Slf4j public class FileHelper { @@ -33,15 +32,17 @@ public class FileHelper { } private static File[] getFileList(File customFolder, String suffix) { - File[] customSubFiles = customFolder.listFiles(file -> { - if (file.isDirectory()) { - return false; - } - if (file.getName().toLowerCase().endsWith(suffix)) { - return true; - } - return false; - }); + File[] customSubFiles = + customFolder.listFiles( + file -> { + if (file.isDirectory()) { + return false; + } + if (file.getName().toLowerCase().endsWith(suffix)) { + return true; + } + return false; + }); return customSubFiles; } @@ -72,8 +73,10 @@ public class FileHelper { log.debug("CustomDictionaryPath:{}", fileList); Config.CustomDictionaryPath = fileList.toArray(new String[0]); - customDictionary.path = (Config.CustomDictionaryPath == null || Config.CustomDictionaryPath.length == 0) ? path - : Config.CustomDictionaryPath; + customDictionary.path = + (Config.CustomDictionaryPath == null || Config.CustomDictionaryPath.length == 0) + ? path + : Config.CustomDictionaryPath; if (Config.CustomDictionaryPath == null || Config.CustomDictionaryPath.length == 0) { Config.CustomDictionaryPath = path; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java index a9759e9ee..4603cd796 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java @@ -33,9 +33,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -/** - * HanLP helper - */ +/** HanLP helper */ @Slf4j public class HanlpHelper { @@ -57,14 +55,21 @@ public class HanlpHelper { if (segment == null) { synchronized (HanlpHelper.class) { if (segment == null) { - segment = HanLP.newSegment() - .enableIndexMode(true).enableIndexMode(4) - .enableCustomDictionary(true).enableCustomDictionaryForcing(true).enableOffset(true) - .enableJapaneseNameRecognize(false).enableNameRecognize(false) - .enableAllNamedEntityRecognize(false) - .enableJapaneseNameRecognize(false).enableNumberQuantifierRecognize(false) - .enablePlaceRecognize(false) - .enableOrganizationRecognize(false).enableCustomDictionary(getDynamicCustomDictionary()); + segment = + HanLP.newSegment() + .enableIndexMode(true) + .enableIndexMode(4) + .enableCustomDictionary(true) + .enableCustomDictionaryForcing(true) + .enableOffset(true) + .enableJapaneseNameRecognize(false) + .enableNameRecognize(false) + .enableAllNamedEntityRecognize(false) + .enableJapaneseNameRecognize(false) + .enableNumberQuantifierRecognize(false) + .enablePlaceRecognize(false) + .enableOrganizationRecognize(false) + .enableCustomDictionary(getDynamicCustomDictionary()); } } } @@ -82,14 +87,13 @@ public class HanlpHelper { return CustomDictionary; } - /*** - * reload custom dictionary - */ + /** * reload custom dictionary */ public static boolean reloadCustomDictionary() throws IOException { final long startTime = System.currentTimeMillis(); - if (HanLP.Config.CustomDictionaryPath == null || HanLP.Config.CustomDictionaryPath.length == 0) { + if (HanLP.Config.CustomDictionaryPath == null + || HanLP.Config.CustomDictionaryPath.length == 0) { return false; } if (HanLP.Config.IOAdapter instanceof HadoopFileIOAdapter) { @@ -106,7 +110,8 @@ public class HanlpHelper { boolean reload = getDynamicCustomDictionary().reload(); if (reload) { - log.info("Custom dictionary has been reloaded in {} milliseconds", + log.info( + "Custom dictionary has been reloaded in {} milliseconds", System.currentTimeMillis() - startTime); } return reload; @@ -118,51 +123,74 @@ public class HanlpHelper { } String hanlpPropertiesPath = getHanlpPropertiesPath(); - HanLP.Config.CustomDictionaryPath = Arrays.stream(HanLP.Config.CustomDictionaryPath) - .map(path -> hanlpPropertiesPath + FILE_SPILT + path) - .toArray(String[]::new); - log.info("hanlpPropertiesPath:{},CustomDictionaryPath:{}", hanlpPropertiesPath, + HanLP.Config.CustomDictionaryPath = + Arrays.stream(HanLP.Config.CustomDictionaryPath) + .map(path -> hanlpPropertiesPath + FILE_SPILT + path) + .toArray(String[]::new); + log.info( + "hanlpPropertiesPath:{},CustomDictionaryPath:{}", + hanlpPropertiesPath, HanLP.Config.CustomDictionaryPath); - HanLP.Config.CoreDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.BiGramDictionaryPath; - HanLP.Config.CoreDictionaryTransformMatrixDictionaryPath = hanlpPropertiesPath + FILE_SPILT - + HanLP.Config.CoreDictionaryTransformMatrixDictionaryPath; - HanLP.Config.BiGramDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.BiGramDictionaryPath; + HanLP.Config.CoreDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.BiGramDictionaryPath; + HanLP.Config.CoreDictionaryTransformMatrixDictionaryPath = + hanlpPropertiesPath + + FILE_SPILT + + HanLP.Config.CoreDictionaryTransformMatrixDictionaryPath; + HanLP.Config.BiGramDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.BiGramDictionaryPath; HanLP.Config.CoreStopWordDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CoreStopWordDictionaryPath; - HanLP.Config.CoreSynonymDictionaryDictionaryPath = hanlpPropertiesPath + FILE_SPILT - + HanLP.Config.CoreSynonymDictionaryDictionaryPath; - HanLP.Config.PersonDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PersonDictionaryPath; - HanLP.Config.PersonDictionaryTrPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PersonDictionaryTrPath; + HanLP.Config.CoreSynonymDictionaryDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CoreSynonymDictionaryDictionaryPath; + HanLP.Config.PersonDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PersonDictionaryPath; + HanLP.Config.PersonDictionaryTrPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PersonDictionaryTrPath; - HanLP.Config.PinyinDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PinyinDictionaryPath; - HanLP.Config.TranslatedPersonDictionaryPath = hanlpPropertiesPath + FILE_SPILT - + HanLP.Config.TranslatedPersonDictionaryPath; - HanLP.Config.JapanesePersonDictionaryPath = hanlpPropertiesPath + FILE_SPILT - + HanLP.Config.JapanesePersonDictionaryPath; - HanLP.Config.PlaceDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PlaceDictionaryPath; - HanLP.Config.PlaceDictionaryTrPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PlaceDictionaryTrPath; - HanLP.Config.OrganizationDictionaryPath = hanlpPropertiesPath + FILE_SPILT - + HanLP.Config.OrganizationDictionaryPath; - HanLP.Config.OrganizationDictionaryTrPath = hanlpPropertiesPath + FILE_SPILT - + HanLP.Config.OrganizationDictionaryTrPath; + HanLP.Config.PinyinDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PinyinDictionaryPath; + HanLP.Config.TranslatedPersonDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.TranslatedPersonDictionaryPath; + HanLP.Config.JapanesePersonDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.JapanesePersonDictionaryPath; + HanLP.Config.PlaceDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PlaceDictionaryPath; + HanLP.Config.PlaceDictionaryTrPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PlaceDictionaryTrPath; + HanLP.Config.OrganizationDictionaryPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.OrganizationDictionaryPath; + HanLP.Config.OrganizationDictionaryTrPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.OrganizationDictionaryTrPath; HanLP.Config.CharTypePath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CharTypePath; HanLP.Config.CharTablePath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CharTablePath; HanLP.Config.PartOfSpeechTagDictionary = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PartOfSpeechTagDictionary; - HanLP.Config.WordNatureModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.WordNatureModelPath; - HanLP.Config.MaxEntModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.MaxEntModelPath; - HanLP.Config.NNParserModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.NNParserModelPath; + HanLP.Config.WordNatureModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.WordNatureModelPath; + HanLP.Config.MaxEntModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.MaxEntModelPath; + HanLP.Config.NNParserModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.NNParserModelPath; HanLP.Config.PerceptronParserModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronParserModelPath; - HanLP.Config.CRFSegmentModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFSegmentModelPath; - HanLP.Config.HMMSegmentModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.HMMSegmentModelPath; - HanLP.Config.CRFCWSModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFCWSModelPath; - HanLP.Config.CRFPOSModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFPOSModelPath; - HanLP.Config.CRFNERModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFNERModelPath; - HanLP.Config.PerceptronCWSModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronCWSModelPath; - HanLP.Config.PerceptronPOSModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronPOSModelPath; - HanLP.Config.PerceptronNERModelPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronNERModelPath; + HanLP.Config.CRFSegmentModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFSegmentModelPath; + HanLP.Config.HMMSegmentModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.HMMSegmentModelPath; + HanLP.Config.CRFCWSModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFCWSModelPath; + HanLP.Config.CRFPOSModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFPOSModelPath; + HanLP.Config.CRFNERModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.CRFNERModelPath; + HanLP.Config.PerceptronCWSModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronCWSModelPath; + HanLP.Config.PerceptronPOSModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronPOSModelPath; + HanLP.Config.PerceptronNERModelPath = + hanlpPropertiesPath + FILE_SPILT + HanLP.Config.PerceptronNERModelPath; } public static String getHanlpPropertiesPath() throws FileNotFoundException { @@ -171,7 +199,8 @@ public class HanlpHelper { public static boolean addToCustomDictionary(DictWord dictWord) { log.debug("dictWord:{}", dictWord); - return getDynamicCustomDictionary().insert(dictWord.getWord(), dictWord.getNatureWithFrequency()); + return getDynamicCustomDictionary() + .insert(dictWord.getWord(), dictWord.getNatureWithFrequency()); } public static void removeFromCustomDictionary(DictWord dictWord) { @@ -195,7 +224,8 @@ public class HanlpHelper { int len = natureWithFrequency.length(); log.info("filtered natureWithFrequency:{}", natureWithFrequency); if (StringUtils.isNotBlank(natureWithFrequency)) { - getDynamicCustomDictionary().add(dictWord.getWord(), natureWithFrequency.substring(0, len - 1)); + getDynamicCustomDictionary() + .add(dictWord.getWord(), natureWithFrequency.substring(0, len - 1)); } SearchService.remove(dictWord, natureList.toArray(new Nature[0])); } @@ -225,8 +255,8 @@ public class HanlpHelper { mapResults.addAll(newResults); } - public static boolean addLetterOriginal(List mapResults, T mapResult, - CoreDictionary.Attribute attribute) { + public static boolean addLetterOriginal( + List mapResults, T mapResult, CoreDictionary.Attribute attribute) { if (attribute == null) { return false; } @@ -236,8 +266,9 @@ public class HanlpHelper { for (String nature : hanlpMapResult.getNatures()) { String orig = attribute.getOriginal(Nature.fromString(nature)); if (orig != null) { - MapResult addMapResult = new HanlpMapResult( - orig, Arrays.asList(nature), hanlpMapResult.getDetectWord()); + MapResult addMapResult = + new HanlpMapResult( + orig, Arrays.asList(nature), hanlpMapResult.getDetectWord()); mapResults.add((T) addMapResult); isAdd = true; } @@ -285,9 +316,12 @@ public class HanlpHelper { .collect(Collectors.toList()); } - public static List transform2ApiTerm(Term term, Map> modelIdToDataSetIds) { + public static List transform2ApiTerm( + Term term, Map> modelIdToDataSetIds) { List s2Terms = Lists.newArrayList(); - List natures = NatureHelper.changeModel2DataSet(String.valueOf(term.getNature()), modelIdToDataSetIds); + List natures = + NatureHelper.changeModel2DataSet( + String.valueOf(term.getNature()), modelIdToDataSetIds); for (String nature : natures) { S2Term s2Term = new S2Term(); BeanUtils.copyProperties(term, s2Term); @@ -297,5 +331,4 @@ public class HanlpHelper { } return s2Terms; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java index eb9192717..0e6d7bad8 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java @@ -3,25 +3,24 @@ package com.tencent.supersonic.headless.chat.knowledge.helper; import com.hankcs.hanlp.HanLP.Config; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.utility.Predefine; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; - import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -/** - * Hdfs File Helper - */ +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +/** Hdfs File Helper */ @Slf4j public class HdfsFileHelper { - /*** - * delete cache file + /** + * * delete cache file + * * @param path * @throws IOException */ @@ -35,7 +34,8 @@ public class HdfsFileHelper { log.error("delete:" + cacheFilePath, e); } int customBase = cacheFilePath.lastIndexOf(FileHelper.FILE_SPILT); - String customPath = cacheFilePath.substring(0, customBase) + FileHelper.FILE_SPILT + "*.bin"; + String customPath = + cacheFilePath.substring(0, customBase) + FileHelper.FILE_SPILT + "*.bin"; List fileList = getFileList(fs, new Path(customPath)); for (String file : fileList) { try { @@ -54,18 +54,22 @@ public class HdfsFileHelper { * @param customDictionary * @throws IOException */ - public static void resetCustomPath(DynamicCustomDictionary customDictionary) throws IOException { + public static void resetCustomPath(DynamicCustomDictionary customDictionary) + throws IOException { String[] path = Config.CustomDictionaryPath; FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration()); String cacheFilePath = path[0] + Predefine.BIN_EXT; int customBase = cacheFilePath.lastIndexOf(FileHelper.FILE_SPILT); - String customPath = cacheFilePath.substring(0, customBase) + FileHelper.FILE_SPILT + "*.txt"; + String customPath = + cacheFilePath.substring(0, customBase) + FileHelper.FILE_SPILT + "*.txt"; log.info("customPath:{}", customPath); List fileList = getFileList(fs, new Path(customPath)); log.info("CustomDictionaryPath:{}", fileList); Config.CustomDictionaryPath = fileList.toArray(new String[0]); - customDictionary.path = (Config.CustomDictionaryPath == null || Config.CustomDictionaryPath.length == 0) ? path - : Config.CustomDictionaryPath; + customDictionary.path = + (Config.CustomDictionaryPath == null || Config.CustomDictionaryPath.length == 0) + ? path + : Config.CustomDictionaryPath; if (Config.CustomDictionaryPath == null || Config.CustomDictionaryPath.length == 0) { Config.CustomDictionaryPath = path; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java index 9499c81a4..94e4c0cf7 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java @@ -5,6 +5,10 @@ import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.chat.knowledge.DataSetInfoStat; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -13,13 +17,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; -/** - * nature parse helper - */ +/** nature parse helper */ @Slf4j public class NatureHelper { @@ -56,8 +55,8 @@ public class NatureHelper { private static boolean isDataSetOrEntity(S2Term term, Integer model) { String natureStr = term.nature.toString(); - return (DictWordType.NATURE_SPILT + model).equals(natureStr) || natureStr.endsWith( - DictWordType.ENTITY.getType()); + return (DictWordType.NATURE_SPILT + model).equals(natureStr) + || natureStr.endsWith(DictWordType.ENTITY.getType()); } public static Integer getDataSetByNature(Nature nature) { @@ -90,7 +89,8 @@ public class NatureHelper { return null; } - public static List changeModel2DataSet(String nature, Map> modelIdToDataSetIds) { + public static List changeModel2DataSet( + String nature, Map> modelIdToDataSetIds) { if (SchemaElementType.TERM.equals(NatureHelper.convertToElementType(nature))) { return Collections.singletonList(nature); } @@ -107,8 +107,10 @@ public class NatureHelper { } public static boolean isDimensionValueDataSetId(String nature) { - return isNatureValid(nature) && !isNatureType(nature, DictWordType.METRIC, DictWordType.DIMENSION, - DictWordType.TERM) && StringUtils.isNumeric(nature.split(DictWordType.NATURE_SPILT)[1]); + return isNatureValid(nature) + && !isNatureType( + nature, DictWordType.METRIC, DictWordType.DIMENSION, DictWordType.TERM) + && StringUtils.isNumeric(nature.split(DictWordType.NATURE_SPILT)[1]); } public static boolean isTermNature(String nature) { @@ -125,34 +127,53 @@ public class NatureHelper { } private static long getDataSetCount(List terms) { - return terms.stream().filter(term -> isDataSetOrEntity(term, getDataSetByNature(term.nature))).count(); + return terms.stream() + .filter(term -> isDataSetOrEntity(term, getDataSetByNature(term.nature))) + .count(); } private static long getDimensionValueCount(List terms) { - return terms.stream().filter(term -> isDimensionValueDataSetId(term.nature.toString())).count(); + return terms.stream() + .filter(term -> isDimensionValueDataSetId(term.nature.toString())) + .count(); } private static long getDimensionCount(List terms) { - return terms.stream().filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT) && term.nature.toString() - .endsWith(DictWordType.DIMENSION.getType())).count(); + return terms.stream() + .filter( + term -> + term.nature.startsWith(DictWordType.NATURE_SPILT) + && term.nature + .toString() + .endsWith(DictWordType.DIMENSION.getType())) + .count(); } private static long getMetricCount(List terms) { - return terms.stream().filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT) && term.nature.toString() - .endsWith(DictWordType.METRIC.getType())).count(); + return terms.stream() + .filter( + term -> + term.nature.startsWith(DictWordType.NATURE_SPILT) + && term.nature + .toString() + .endsWith(DictWordType.METRIC.getType())) + .count(); } public static Map> getDataSetToNatureStat(List terms) { Map> modelToNature = new HashMap<>(); terms.stream() .filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT)) - .forEach(term -> { - DictWordType dictWordType = DictWordType.getNatureType(term.nature.toString()); - Long model = getDataSetId(term.nature.toString()); + .forEach( + term -> { + DictWordType dictWordType = + DictWordType.getNatureType(term.nature.toString()); + Long model = getDataSetId(term.nature.toString()); - modelToNature.computeIfAbsent(model, k -> new HashMap<>()) - .merge(dictWordType, 1, Integer::sum); - }); + modelToNature + .computeIfAbsent(model, k -> new HashMap<>()) + .merge(dictWordType, 1, Integer::sum); + }); return modelToNature; } @@ -160,10 +181,12 @@ public class NatureHelper { Map> modelToNatureStat = getDataSetToNatureStat(terms); return modelToNatureStat.entrySet().stream() .max(Comparator.comparingInt(entry -> entry.getValue().size())) - .map(entry -> modelToNatureStat.entrySet().stream() - .filter(e -> e.getValue().size() == entry.getValue().size()) - .map(Map.Entry::getKey) - .collect(Collectors.toList())) + .map( + entry -> + modelToNatureStat.entrySet().stream() + .filter(e -> e.getValue().size() == entry.getValue().size()) + .map(Map.Entry::getKey) + .collect(Collectors.toList())) .orElse(Collections.emptyList()); } @@ -171,7 +194,8 @@ public class NatureHelper { return parseIdFromNature(nature, 2); } - public static Set getModelIds(Map> modelIdToDataSetIds, Set detectDataSetIds) { + public static Set getModelIds( + Map> modelIdToDataSetIds, Set detectDataSetIds) { if (CollectionUtils.isEmpty(detectDataSetIds)) { return modelIdToDataSetIds.keySet(); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMapper.java index 0a1b6fa9e..a718756af 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMapper.java @@ -13,11 +13,11 @@ import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.HashSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -30,7 +30,9 @@ public abstract class BaseMapper implements SchemaMapper { String simpleName = this.getClass().getSimpleName(); long startTime = System.currentTimeMillis(); - log.debug("before {},mapInfo:{}", simpleName, + log.debug( + "before {},mapInfo:{}", + simpleName, chatQueryContext.getMapInfo().getDataSetElementMatches()); try { @@ -41,7 +43,10 @@ public abstract class BaseMapper implements SchemaMapper { } long cost = System.currentTimeMillis() - startTime; - log.debug("after {},cost:{},mapInfo:{}", simpleName, cost, + log.debug( + "after {},cost:{},mapInfo:{}", + simpleName, + cost, chatQueryContext.getMapInfo().getDataSetElementMatches()); } @@ -53,14 +58,19 @@ public abstract class BaseMapper implements SchemaMapper { filterByQueryDataType(chatQueryContext, element -> !(element.getIsTag() > 0)); break; case METRIC: - filterByQueryDataType(chatQueryContext, element -> !SchemaElementType.METRIC.equals(element.getType())); + filterByQueryDataType( + chatQueryContext, + element -> !SchemaElementType.METRIC.equals(element.getType())); break; case DIMENSION: - filterByQueryDataType(chatQueryContext, element -> { - boolean isDimensionOrValue = SchemaElementType.DIMENSION.equals(element.getType()) - || SchemaElementType.VALUE.equals(element.getType()); - return !isDimensionOrValue; - }); + filterByQueryDataType( + chatQueryContext, + element -> { + boolean isDimensionOrValue = + SchemaElementType.DIMENSION.equals(element.getType()) + || SchemaElementType.VALUE.equals(element.getType()); + return !isDimensionOrValue; + }); break; case ALL: default: @@ -73,7 +83,8 @@ public abstract class BaseMapper implements SchemaMapper { if (CollectionUtils.isEmpty(dataSetIds)) { return; } - Set dataSetIdInMapInfo = new HashSet<>(chatQueryContext.getMapInfo().getDataSetElementMatches().keySet()); + Set dataSetIdInMapInfo = + new HashSet<>(chatQueryContext.getMapInfo().getDataSetElementMatches().keySet()); for (Long dataSetId : dataSetIdInMapInfo) { if (!dataSetIds.contains(dataSetId)) { chatQueryContext.getMapInfo().getDataSetElementMatches().remove(dataSetId); @@ -87,54 +98,68 @@ public abstract class BaseMapper implements SchemaMapper { for (Map.Entry> entry : dataSetElementMatches.entrySet()) { List value = entry.getValue(); if (!CollectionUtils.isEmpty(value)) { - value.removeIf(schemaElementMatch -> - StringUtils.length(schemaElementMatch.getDetectWord()) <= 1); + value.removeIf( + schemaElementMatch -> + StringUtils.length(schemaElementMatch.getDetectWord()) <= 1); } } } - private static void filterByQueryDataType(ChatQueryContext chatQueryContext, - Predicate needRemovePredicate) { - chatQueryContext.getMapInfo().getDataSetElementMatches().values().forEach(schemaElementMatches -> { - schemaElementMatches.removeIf(schemaElementMatch -> { - SchemaElement element = schemaElementMatch.getElement(); - SchemaElementType type = element.getType(); + private static void filterByQueryDataType( + ChatQueryContext chatQueryContext, Predicate needRemovePredicate) { + chatQueryContext + .getMapInfo() + .getDataSetElementMatches() + .values() + .forEach( + schemaElementMatches -> { + schemaElementMatches.removeIf( + schemaElementMatch -> { + SchemaElement element = schemaElementMatch.getElement(); + SchemaElementType type = element.getType(); - boolean isEntityOrDatasetOrId = SchemaElementType.ENTITY.equals(type) - || SchemaElementType.DATASET.equals(type) - || SchemaElementType.ID.equals(type); + boolean isEntityOrDatasetOrId = + SchemaElementType.ENTITY.equals(type) + || SchemaElementType.DATASET.equals(type) + || SchemaElementType.ID.equals(type); - return !isEntityOrDatasetOrId && needRemovePredicate.test(element); - }); - }); + return !isEntityOrDatasetOrId + && needRemovePredicate.test(element); + }); + }); } public abstract void doMap(ChatQueryContext chatQueryContext); - public void addToSchemaMap(SchemaMapInfo schemaMap, Long dataSetId, SchemaElementMatch newElementMatch) { - Map> dataSetElementMatches = schemaMap.getDataSetElementMatches(); - List schemaElementMatches = dataSetElementMatches.computeIfAbsent(dataSetId, - k -> new ArrayList<>()); + public void addToSchemaMap( + SchemaMapInfo schemaMap, Long dataSetId, SchemaElementMatch newElementMatch) { + Map> dataSetElementMatches = + schemaMap.getDataSetElementMatches(); + List schemaElementMatches = + dataSetElementMatches.computeIfAbsent(dataSetId, k -> new ArrayList<>()); AtomicBoolean shouldAddNew = new AtomicBoolean(true); - schemaElementMatches.removeIf(existingElementMatch -> { - if (isEquals(existingElementMatch, newElementMatch)) { - if (newElementMatch.getSimilarity() > existingElementMatch.getSimilarity()) { - return true; - } else { - shouldAddNew.set(false); - } - } - return false; - }); + schemaElementMatches.removeIf( + existingElementMatch -> { + if (isEquals(existingElementMatch, newElementMatch)) { + if (newElementMatch.getSimilarity() + > existingElementMatch.getSimilarity()) { + return true; + } else { + shouldAddNew.set(false); + } + } + return false; + }); if (shouldAddNew.get()) { schemaElementMatches.add(newElementMatch); } } - private static boolean isEquals(SchemaElementMatch existElementMatch, SchemaElementMatch newElementMatch) { + private static boolean isEquals( + SchemaElementMatch existElementMatch, SchemaElementMatch newElementMatch) { SchemaElement existElement = existElementMatch.getElement(); SchemaElement newElement = newElementMatch.getElement(); if (!existElement.equals(newElement)) { @@ -146,8 +171,11 @@ public abstract class BaseMapper implements SchemaMapper { return true; } - public SchemaElement getSchemaElement(Long dataSetId, SchemaElementType elementType, Long elementID, - SemanticSchema semanticSchema) { + public SchemaElement getSchemaElement( + Long dataSetId, + SchemaElementType elementType, + Long elementID, + SemanticSchema semanticSchema) { SchemaElement element = new SchemaElement(); DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId); if (Objects.isNull(dataSetSchema)) { @@ -166,8 +194,8 @@ public abstract class BaseMapper implements SchemaMapper { if (!SchemaElementType.VALUE.equals(element.getType())) { return element.getAlias(); } - if (org.apache.commons.collections.CollectionUtils.isNotEmpty(element.getAlias()) && StringUtils.isNotEmpty( - element.getName())) { + if (org.apache.commons.collections.CollectionUtils.isNotEmpty(element.getAlias()) + && StringUtils.isNotEmpty(element.getName())) { return element.getAlias().stream() .filter(aliasItem -> aliasItem.contains(element.getName())) .collect(Collectors.toList()); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java index 1b49507f9..d6cb1abd1 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.mapper; - import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.chat.ChatQueryContext; @@ -26,15 +25,13 @@ import java.util.stream.Collectors; @Slf4j public abstract class BaseMatchStrategy implements MatchStrategy { - @Autowired - protected MapperHelper mapperHelper; + @Autowired protected MapperHelper mapperHelper; - @Autowired - protected MapperConfig mapperConfig; + @Autowired protected MapperConfig mapperConfig; @Override - public Map> match(ChatQueryContext chatQueryContext, List terms, - Set detectDataSetIds) { + public Map> match( + ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds) { String text = chatQueryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; @@ -49,7 +46,8 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return result; } - public List detect(ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds) { + public List detect( + ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds) { Map regOffsetToLength = getRegOffsetToLength(terms); String text = chatQueryContext.getQueryText(); Set results = new HashSet<>(); @@ -64,7 +62,8 @@ public abstract class BaseMatchStrategy implements MatchStrategy { if (index <= text.length()) { String detectSegment = text.substring(startIndex, index).trim(); detectSegments.add(detectSegment); - detectByStep(chatQueryContext, results, detectDataSetIds, detectSegment, offset); + detectByStep( + chatQueryContext, results, detectDataSetIds, detectSegment, offset); } } startIndex = mapperHelper.getStepIndex(regOffsetToLength, startIndex); @@ -73,9 +72,13 @@ public abstract class BaseMatchStrategy implements MatchStrategy { } public Map getRegOffsetToLength(List terms) { - return terms.stream().sorted(Comparator.comparing(S2Term::length)) - .collect(Collectors.toMap(S2Term::getOffset, term -> term.word.length(), - (value1, value2) -> value2)); + return terms.stream() + .sorted(Comparator.comparing(S2Term::length)) + .collect( + Collectors.toMap( + S2Term::getOffset, + term -> term.word.length(), + (value1, value2) -> value2)); } public void selectResultInOneRound(Set existResults, List oneRoundResults) { @@ -84,15 +87,15 @@ public abstract class BaseMatchStrategy implements MatchStrategy { } for (T oneRoundResult : oneRoundResults) { if (existResults.contains(oneRoundResult)) { - boolean isDeleted = existResults.removeIf( - existResult -> { - boolean delete = needDelete(oneRoundResult, existResult); - if (delete) { - log.info("deleted existResult:{}", existResult); - } - return delete; - } - ); + boolean isDeleted = + existResults.removeIf( + existResult -> { + boolean delete = needDelete(oneRoundResult, existResult); + if (delete) { + log.info("deleted existResult:{}", existResult); + } + return delete; + }); if (isDeleted) { log.info("deleted, add oneRoundResult:{}", oneRoundResult); existResults.add(oneRoundResult); @@ -111,9 +114,11 @@ public abstract class BaseMatchStrategy implements MatchStrategy { if (Objects.isNull(matchResult)) { return matches; } - Optional> first = matchResult.entrySet().stream() - .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue())) - .map(entry -> entry.getValue()).findFirst(); + Optional> first = + matchResult.entrySet().stream() + .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue())) + .map(entry -> entry.getValue()) + .findFirst(); if (first.isPresent()) { matches = first.get(); @@ -124,13 +129,19 @@ public abstract class BaseMatchStrategy implements MatchStrategy { public List filterByDataSetId(List terms, Set dataSetIds) { logTerms(terms); if (CollectionUtils.isNotEmpty(dataSetIds)) { - terms = terms.stream().filter(term -> { - Long dataSetId = NatureHelper.getDataSetId(term.getNature().toString()); - if (Objects.nonNull(dataSetId)) { - return dataSetIds.contains(dataSetId); - } - return false; - }).collect(Collectors.toList()); + terms = + terms.stream() + .filter( + term -> { + Long dataSetId = + NatureHelper.getDataSetId( + term.getNature().toString()); + if (Objects.nonNull(dataSetId)) { + return dataSetIds.contains(dataSetId); + } + return false; + }) + .collect(Collectors.toList()); log.debug("terms filter by dataSetId:{}", dataSetIds); logTerms(terms); } @@ -142,7 +153,11 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return; } for (S2Term term : terms) { - log.debug("word:{},nature:{},frequency:{}", term.word, term.nature.toString(), term.getFrequency()); + log.debug( + "word:{},nature:{},frequency:{}", + term.word, + term.nature.toString(), + term.getFrequency()); } } @@ -150,8 +165,12 @@ public abstract class BaseMatchStrategy implements MatchStrategy { public abstract String getMapKey(T a); - public abstract void detectByStep(ChatQueryContext chatQueryContext, Set existResults, - Set detectDataSetIds, String detectSegment, int offset); + public abstract void detectByStep( + ChatQueryContext chatQueryContext, + Set existResults, + Set detectDataSetIds, + String detectSegment, + int offset); public double getThreshold(Double threshold, Double minThreshold, MapModeEnum mapModeEnum) { double decreaseAmount = (threshold - minThreshold) / 4; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/DatabaseMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/DatabaseMatchStrategy.java index 9bbca97a2..f9dd9d6c1 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/DatabaseMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/DatabaseMatchStrategy.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.mapper; - import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; @@ -21,8 +20,8 @@ import java.util.Set; import java.util.stream.Collectors; /** - * DatabaseMatchStrategy uses SQL LIKE operator to match schema elements. - * It currently supports fuzzy matching against names and aliases. + * DatabaseMatchStrategy uses SQL LIKE operator to match schema elements. It currently supports + * fuzzy matching against names and aliases. */ @Service @Slf4j @@ -31,8 +30,8 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy private List allElements; @Override - public Map> match(ChatQueryContext chatQueryContext, List terms, - Set detectDataSetIds) { + public Map> match( + ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds) { this.allElements = getSchemaElements(chatQueryContext); return super.match(chatQueryContext, terms, detectDataSetIds); } @@ -45,12 +44,19 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy @Override public String getMapKey(DatabaseMapResult a) { - return a.getName() + Constants.UNDERLINE + a.getSchemaElement().getId() - + Constants.UNDERLINE + a.getSchemaElement().getName(); + return a.getName() + + Constants.UNDERLINE + + a.getSchemaElement().getId() + + Constants.UNDERLINE + + a.getSchemaElement().getName(); } - public void detectByStep(ChatQueryContext chatQueryContext, Set existResults, - Set detectDataSetIds, String detectSegment, int offset) { + public void detectByStep( + ChatQueryContext chatQueryContext, + Set existResults, + Set detectDataSetIds, + String detectSegment, + int offset) { if (StringUtils.isBlank(detectSegment)) { return; } @@ -61,14 +67,19 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy for (Entry> entry : nameToItems.entrySet()) { String name = entry.getKey(); if (!name.contains(detectSegment) - || mapperHelper.getSimilarity(detectSegment, name) < metricDimensionThresholdConfig) { + || mapperHelper.getSimilarity(detectSegment, name) + < metricDimensionThresholdConfig) { continue; } Set schemaElements = entry.getValue(); if (!CollectionUtils.isEmpty(detectDataSetIds)) { - schemaElements = schemaElements.stream() - .filter(schemaElement -> detectDataSetIds.contains(schemaElement.getDataSetId())) - .collect(Collectors.toSet()); + schemaElements = + schemaElements.stream() + .filter( + schemaElement -> + detectDataSetIds.contains( + schemaElement.getDataSetId())) + .collect(Collectors.toSet()); } for (SchemaElement schemaElement : schemaElements) { DatabaseMapResult databaseMapResult = new DatabaseMapResult(); @@ -88,31 +99,42 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy } private Double getThreshold(ChatQueryContext chatQueryContext) { - Double threshold = Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD)); - Double minThreshold = Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD_MIN)); + Double threshold = + Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD)); + Double minThreshold = + Double.valueOf( + mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD_MIN)); - Map> modelElementMatches = chatQueryContext.getMapInfo() - .getDataSetElementMatches(); + Map> modelElementMatches = + chatQueryContext.getMapInfo().getDataSetElementMatches(); - boolean existElement = modelElementMatches.entrySet().stream().anyMatch(entry -> entry.getValue().size() >= 1); + boolean existElement = + modelElementMatches.entrySet().stream() + .anyMatch(entry -> entry.getValue().size() >= 1); if (!existElement) { threshold = threshold / 2; - log.debug("ModelElementMatches:{},not exist Element threshold reduce by half:{}", - modelElementMatches, threshold); + log.debug( + "ModelElementMatches:{},not exist Element threshold reduce by half:{}", + modelElementMatches, + threshold); } return getThreshold(threshold, minThreshold, chatQueryContext.getMapModeEnum()); } private Map> getNameToItems(List models) { - return models.stream().collect( - Collectors.toMap(SchemaElement::getName, a -> { - Set result = new HashSet<>(); - result.add(a); - return result; - }, (k1, k2) -> { - k1.addAll(k2); - return k1; - })); + return models.stream() + .collect( + Collectors.toMap( + SchemaElement::getName, + a -> { + Set result = new HashSet<>(); + result.add(a); + return result; + }, + (k1, k2) -> { + k1.addAll(k2); + return k1; + })); } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMapper.java index 196343a48..44adfc396 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMapper.java @@ -1,59 +1,63 @@ package com.tencent.supersonic.headless.chat.mapper; - import com.tencent.supersonic.common.util.ContextUtils; -import dev.langchain4j.store.embedding.Retrieval; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.knowledge.EmbeddingResult; import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; -import com.tencent.supersonic.headless.chat.ChatQueryContext; +import dev.langchain4j.store.embedding.Retrieval; import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.Objects; -/*** - * A mapper that recognizes schema elements with vector embedding. - */ +/** * A mapper that recognizes schema elements with vector embedding. */ @Slf4j public class EmbeddingMapper extends BaseMapper { @Override public void doMap(ChatQueryContext chatQueryContext) { - //1. query from embedding by queryText + // 1. query from embedding by queryText String queryText = chatQueryContext.getQueryText(); - List terms = HanlpHelper.getTerms(queryText, chatQueryContext.getModelIdToDataSetIds()); + List terms = + HanlpHelper.getTerms(queryText, chatQueryContext.getModelIdToDataSetIds()); EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class); List matchResults = matchStrategy.getMatches(chatQueryContext, terms); HanlpHelper.transLetterOriginal(matchResults); - //2. build SchemaElementMatch by info + // 2. build SchemaElementMatch by info for (EmbeddingResult matchResult : matchResults) { Long elementId = Retrieval.getLongId(matchResult.getId()); Long dataSetId = Retrieval.getLongId(matchResult.getMetadata().get("dataSetId")); if (Objects.isNull(dataSetId)) { continue; } - SchemaElementType elementType = SchemaElementType.valueOf(matchResult.getMetadata().get("type")); - SchemaElement schemaElement = getSchemaElement(dataSetId, elementType, elementId, - chatQueryContext.getSemanticSchema()); + SchemaElementType elementType = + SchemaElementType.valueOf(matchResult.getMetadata().get("type")); + SchemaElement schemaElement = + getSchemaElement( + dataSetId, + elementType, + elementId, + chatQueryContext.getSemanticSchema()); if (schemaElement == null) { continue; } - SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() - .element(schemaElement) - .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) - .word(matchResult.getName()) - .similarity(1 - matchResult.getDistance()) - .detectWord(matchResult.getDetectWord()) - .build(); - //3. add to mapInfo + SchemaElementMatch schemaElementMatch = + SchemaElementMatch.builder() + .element(schemaElement) + .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) + .word(matchResult.getName()) + .similarity(1 - matchResult.getDistance()) + .detectWord(matchResult.getDetectWord()) + .build(); + // 3. add to mapInfo addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch); } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java index d9b492d66..8ebffa519 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java @@ -3,12 +3,12 @@ package com.tencent.supersonic.headless.chat.mapper; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.headless.api.pojo.response.S2Term; -import dev.langchain4j.store.embedding.Retrieval; -import dev.langchain4j.store.embedding.RetrieveQuery; -import dev.langchain4j.store.embedding.RetrieveQueryResult; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.knowledge.EmbeddingResult; import com.tencent.supersonic.headless.chat.knowledge.MetaEmbeddingService; +import dev.langchain4j.store.embedding.Retrieval; +import dev.langchain4j.store.embedding.RetrieveQuery; +import dev.langchain4j.store.embedding.RetrieveQueryResult; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -30,15 +30,14 @@ import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.EMBEDDING import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.EMBEDDING_MAPPER_THRESHOLD_MIN; /** - * EmbeddingMatchStrategy uses vector database to perform - * similarity search against the embeddings of schema elements. + * EmbeddingMatchStrategy uses vector database to perform similarity search against the embeddings + * of schema elements. */ @Service @Slf4j public class EmbeddingMatchStrategy extends BaseMatchStrategy { - @Autowired - private MetaEmbeddingService metaEmbeddingService; + @Autowired private MetaEmbeddingService metaEmbeddingService; @Override public boolean needDelete(EmbeddingResult oneRoundResult, EmbeddingResult existResult) { @@ -52,44 +51,54 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { } @Override - public void detectByStep(ChatQueryContext chatQueryContext, Set existResults, - Set detectDataSetIds, String detectSegment, int offset) { - - } + public void detectByStep( + ChatQueryContext chatQueryContext, + Set existResults, + Set detectDataSetIds, + String detectSegment, + int offset) {} @Override - public List detect(ChatQueryContext chatQueryContext, List terms, - Set detectDataSetIds) { + public List detect( + ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds) { String text = chatQueryContext.getQueryText(); Set detectSegments = new HashSet<>(); - int embeddingTextSize = Integer.valueOf( - mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_TEXT_SIZE)); + int embeddingTextSize = + Integer.valueOf( + mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_TEXT_SIZE)); - int embeddingTextStep = Integer.valueOf( - mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_TEXT_STEP)); + int embeddingTextStep = + Integer.valueOf( + mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_TEXT_STEP)); for (int startIndex = 0; startIndex < text.length(); startIndex += embeddingTextStep) { int endIndex = Math.min(startIndex + embeddingTextSize, text.length()); String detectSegment = text.substring(startIndex, endIndex).trim(); detectSegments.add(detectSegment); } - Set results = detectByBatch(chatQueryContext, detectDataSetIds, detectSegments); + Set results = + detectByBatch(chatQueryContext, detectDataSetIds, detectSegments); return new ArrayList<>(results); } - protected Set detectByBatch(ChatQueryContext chatQueryContext, - Set detectDataSetIds, Set detectSegments) { + protected Set detectByBatch( + ChatQueryContext chatQueryContext, + Set detectDataSetIds, + Set detectSegments) { Set results = new HashSet<>(); - int embeddingMapperBatch = Integer.valueOf(mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_BATCH)); + int embeddingMapperBatch = + Integer.valueOf( + mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_BATCH)); - List queryTextsList = detectSegments.stream() - .map(detectSegment -> detectSegment.trim()) - .filter(detectSegment -> StringUtils.isNotBlank(detectSegment)) - .collect(Collectors.toList()); + List queryTextsList = + detectSegments.stream() + .map(detectSegment -> detectSegment.trim()) + .filter(detectSegment -> StringUtils.isNotBlank(detectSegment)) + .collect(Collectors.toList()); - List> queryTextsSubList = Lists.partition(queryTextsList, - embeddingMapperBatch); + List> queryTextsSubList = + Lists.partition(queryTextsList, embeddingMapperBatch); for (List queryTextsSub : queryTextsSubList) { detectByQueryTextsSub(results, detectDataSetIds, queryTextsSub, chatQueryContext); @@ -97,60 +106,99 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { return results; } - private void detectByQueryTextsSub(Set results, Set detectDataSetIds, - List queryTextsSub, ChatQueryContext chatQueryContext) { + private void detectByQueryTextsSub( + Set results, + Set detectDataSetIds, + List queryTextsSub, + ChatQueryContext chatQueryContext) { Map> modelIdToDataSetIds = chatQueryContext.getModelIdToDataSetIds(); - double embeddingThreshold = Double.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_THRESHOLD)); - double embeddingThresholdMin = Double.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_THRESHOLD_MIN)); - double threshold = getThreshold(embeddingThreshold, embeddingThresholdMin, chatQueryContext.getMapModeEnum()); + double embeddingThreshold = + Double.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_THRESHOLD)); + double embeddingThresholdMin = + Double.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_THRESHOLD_MIN)); + double threshold = + getThreshold( + embeddingThreshold, + embeddingThresholdMin, + chatQueryContext.getMapModeEnum()); // step1. build query params RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(queryTextsSub).build(); // step2. retrieveQuery by detectSegment - int embeddingNumber = Integer.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_NUMBER)); - List retrieveQueryResults = metaEmbeddingService.retrieveQuery( - retrieveQuery, embeddingNumber, modelIdToDataSetIds, detectDataSetIds); + int embeddingNumber = + Integer.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_NUMBER)); + List retrieveQueryResults = + metaEmbeddingService.retrieveQuery( + retrieveQuery, embeddingNumber, modelIdToDataSetIds, detectDataSetIds); if (CollectionUtils.isEmpty(retrieveQueryResults)) { return; } // step3. build EmbeddingResults - List collect = retrieveQueryResults.stream() - .map(retrieveQueryResult -> { - List retrievals = retrieveQueryResult.getRetrieval(); - if (CollectionUtils.isNotEmpty(retrievals)) { - retrievals.removeIf(retrieval -> { - if (!retrieveQueryResult.getQuery().contains(retrieval.getQuery())) { - return retrieval.getDistance() > 1 - threshold; - } - return false; - }); - } - return retrieveQueryResult; - }) - .filter(retrieveQueryResult -> CollectionUtils.isNotEmpty(retrieveQueryResult.getRetrieval())) - .flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream() - .map(retrieval -> { - EmbeddingResult embeddingResult = new EmbeddingResult(); - BeanUtils.copyProperties(retrieval, embeddingResult); - embeddingResult.setDetectWord(retrieveQueryResult.getQuery()); - embeddingResult.setName(retrieval.getQuery()); - Map convertedMap = retrieval.getMetadata().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toString())); - embeddingResult.setMetadata(convertedMap); - return embeddingResult; - })) - .collect(Collectors.toList()); + List collect = + retrieveQueryResults.stream() + .map( + retrieveQueryResult -> { + List retrievals = retrieveQueryResult.getRetrieval(); + if (CollectionUtils.isNotEmpty(retrievals)) { + retrievals.removeIf( + retrieval -> { + if (!retrieveQueryResult + .getQuery() + .contains(retrieval.getQuery())) { + return retrieval.getDistance() + > 1 - threshold; + } + return false; + }); + } + return retrieveQueryResult; + }) + .filter( + retrieveQueryResult -> + CollectionUtils.isNotEmpty( + retrieveQueryResult.getRetrieval())) + .flatMap( + retrieveQueryResult -> + retrieveQueryResult.getRetrieval().stream() + .map( + retrieval -> { + EmbeddingResult embeddingResult = + new EmbeddingResult(); + BeanUtils.copyProperties( + retrieval, embeddingResult); + embeddingResult.setDetectWord( + retrieveQueryResult.getQuery()); + embeddingResult.setName( + retrieval.getQuery()); + Map convertedMap = + retrieval.getMetadata() + .entrySet().stream() + .collect( + Collectors + .toMap( + Map + .Entry + ::getKey, + entry -> + entry.getValue() + .toString())); + embeddingResult.setMetadata( + convertedMap); + return embeddingResult; + })) + .collect(Collectors.toList()); // step4. select mapResul in one round - int embeddingRoundNumber = Integer.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_ROUND_NUMBER)); + int embeddingRoundNumber = + Integer.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_ROUND_NUMBER)); int roundNumber = embeddingRoundNumber * queryTextsSub.size(); - List oneRoundResults = collect.stream() - .sorted(Comparator.comparingDouble(EmbeddingResult::getDistance)) - .limit(roundNumber) - .collect(Collectors.toList()); + List oneRoundResults = + collect.stream() + .sorted(Comparator.comparingDouble(EmbeddingResult::getDistance)) + .limit(roundNumber) + .collect(Collectors.toList()); selectResultInOneRound(results, oneRoundResults); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java index 9137784b8..6b1a7f7aa 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java @@ -14,9 +14,7 @@ import org.springframework.util.CollectionUtils; import java.util.List; import java.util.stream.Collectors; -/** - * A mapper capable of converting the VALUE of entity dimension values into ID types. - */ +/** A mapper capable of converting the VALUE of entity dimension values into ID types. */ @Slf4j public class EntityMapper extends BaseMapper { @@ -24,7 +22,8 @@ public class EntityMapper extends BaseMapper { public void doMap(ChatQueryContext chatQueryContext) { SchemaMapInfo schemaMapInfo = chatQueryContext.getMapInfo(); for (Long dataSetId : schemaMapInfo.getMatchedDataSetInfos()) { - List schemaElementMatchList = schemaMapInfo.getMatchedElements(dataSetId); + List schemaElementMatchList = + schemaMapInfo.getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(schemaElementMatchList)) { continue; } @@ -32,15 +31,19 @@ public class EntityMapper extends BaseMapper { if (entity == null || entity.getId() == null) { continue; } - List valueSchemaElements = schemaElementMatchList.stream() - .filter(schemaElementMatch -> SchemaElementType.VALUE.equals( - schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); + List valueSchemaElements = + schemaElementMatchList.stream() + .filter( + schemaElementMatch -> + SchemaElementType.VALUE.equals( + schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); for (SchemaElementMatch schemaElementMatch : valueSchemaElements) { if (!entity.getId().equals(schemaElementMatch.getElement().getId())) { continue; } - if (!checkExistSameEntitySchemaElements(schemaElementMatch, schemaElementMatchList)) { + if (!checkExistSameEntitySchemaElements( + schemaElementMatch, schemaElementMatchList)) { SchemaElementMatch entitySchemaElementMath = new SchemaElementMatch(); BeanUtils.copyProperties(schemaElementMatch, entitySchemaElementMath); entitySchemaElementMath.setElement(entity); @@ -51,13 +54,21 @@ public class EntityMapper extends BaseMapper { } } - private boolean checkExistSameEntitySchemaElements(SchemaElementMatch valueSchemaElementMatch, + private boolean checkExistSameEntitySchemaElements( + SchemaElementMatch valueSchemaElementMatch, List schemaElementMatchList) { - List entitySchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch -> - SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); + List entitySchemaElements = + schemaElementMatchList.stream() + .filter( + schemaElementMatch -> + SchemaElementType.ENTITY.equals( + schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); for (SchemaElementMatch schemaElementMatch : entitySchemaElements) { - if (schemaElementMatch.getElement().getId().equals(valueSchemaElementMatch.getElement().getId())) { + if (schemaElementMatch + .getElement() + .getId() + .equals(valueSchemaElementMatch.getElement().getId())) { return true; } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/HanlpDictMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/HanlpDictMatchStrategy.java index f0e2cac07..11cb16383 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/HanlpDictMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/HanlpDictMatchStrategy.java @@ -25,20 +25,18 @@ import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.MAPPER_DE import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.MAPPER_DIMENSION_VALUE_SIZE; /** - * HanlpDictMatchStrategy uses HanLP to - * match schema elements. It currently supports prefix and suffix matching - * against names, values and aliases. + * HanlpDictMatchStrategy uses HanLP to match schema elements. + * It currently supports prefix and suffix matching against names, values and aliases. */ @Service @Slf4j public class HanlpDictMatchStrategy extends BaseMatchStrategy { - @Autowired - private KnowledgeBaseService knowledgeBaseService; + @Autowired private KnowledgeBaseService knowledgeBaseService; @Override - public Map> match(ChatQueryContext chatQueryContext, List terms, - Set detectDataSetIds) { + public Map> match( + ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds) { String text = chatQueryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; @@ -59,18 +57,34 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { && existResult.getDetectWord().length() < oneRoundResult.getDetectWord().length(); } - public void detectByStep(ChatQueryContext chatQueryContext, Set existResults, - Set detectDataSetIds, - String detectSegment, int offset) { + public void detectByStep( + ChatQueryContext chatQueryContext, + Set existResults, + Set detectDataSetIds, + String detectSegment, + int offset) { // step1. pre search - Integer oneDetectionMaxSize = Integer.valueOf(mapperConfig.getParameterValue(MAPPER_DETECTION_MAX_SIZE)); - LinkedHashSet hanlpMapResults = knowledgeBaseService.prefixSearch(detectSegment, - oneDetectionMaxSize, chatQueryContext.getModelIdToDataSetIds(), detectDataSetIds) - .stream().collect(Collectors.toCollection(LinkedHashSet::new)); + Integer oneDetectionMaxSize = + Integer.valueOf(mapperConfig.getParameterValue(MAPPER_DETECTION_MAX_SIZE)); + LinkedHashSet hanlpMapResults = + knowledgeBaseService + .prefixSearch( + detectSegment, + oneDetectionMaxSize, + chatQueryContext.getModelIdToDataSetIds(), + detectDataSetIds) + .stream() + .collect(Collectors.toCollection(LinkedHashSet::new)); // step2. suffix search - LinkedHashSet suffixHanlpMapResults = knowledgeBaseService.suffixSearch(detectSegment, - oneDetectionMaxSize, chatQueryContext.getModelIdToDataSetIds(), detectDataSetIds) - .stream().collect(Collectors.toCollection(LinkedHashSet::new)); + LinkedHashSet suffixHanlpMapResults = + knowledgeBaseService + .suffixSearch( + detectSegment, + oneDetectionMaxSize, + chatQueryContext.getModelIdToDataSetIds(), + detectDataSetIds) + .stream() + .collect(Collectors.toCollection(LinkedHashSet::new)); hanlpMapResults.addAll(suffixHanlpMapResults); @@ -78,45 +92,67 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { return; } // step3. merge pre/suffix result - hanlpMapResults = hanlpMapResults.stream().sorted((a, b) -> -(b.getName().length() - a.getName().length())) - .collect(Collectors.toCollection(LinkedHashSet::new)); + hanlpMapResults = + hanlpMapResults.stream() + .sorted((a, b) -> -(b.getName().length() - a.getName().length())) + .collect(Collectors.toCollection(LinkedHashSet::new)); // step4. filter by similarity - hanlpMapResults = hanlpMapResults.stream() - .filter(term -> mapperHelper.getSimilarity(detectSegment, term.getName()) - >= getThresholdMatch(term.getNatures(), chatQueryContext)) - .filter(term -> CollectionUtils.isNotEmpty(term.getNatures())) - .collect(Collectors.toCollection(LinkedHashSet::new)); + hanlpMapResults = + hanlpMapResults.stream() + .filter( + term -> + mapperHelper.getSimilarity(detectSegment, term.getName()) + >= getThresholdMatch( + term.getNatures(), chatQueryContext)) + .filter(term -> CollectionUtils.isNotEmpty(term.getNatures())) + .collect(Collectors.toCollection(LinkedHashSet::new)); - log.debug("detectSegment:{},after isSimilarity parseResults:{}", detectSegment, hanlpMapResults); + log.debug( + "detectSegment:{},after isSimilarity parseResults:{}", + detectSegment, + hanlpMapResults); - hanlpMapResults = hanlpMapResults.stream().map(parseResult -> { - parseResult.setOffset(offset); - parseResult.setSimilarity(mapperHelper.getSimilarity(detectSegment, parseResult.getName())); - return parseResult; - }).collect(Collectors.toCollection(LinkedHashSet::new)); + hanlpMapResults = + hanlpMapResults.stream() + .map( + parseResult -> { + parseResult.setOffset(offset); + parseResult.setSimilarity( + mapperHelper.getSimilarity( + detectSegment, parseResult.getName())); + return parseResult; + }) + .collect(Collectors.toCollection(LinkedHashSet::new)); // step5. take only M dimensionValue or N-M metric/dimension value per rond. - int oneDetectionValueSize = Integer.valueOf(mapperConfig.getParameterValue(MAPPER_DIMENSION_VALUE_SIZE)); - List dimensionValues = hanlpMapResults.stream() - .filter(entry -> mapperHelper.existDimensionValues(entry.getNatures())) - .limit(oneDetectionValueSize) - .collect(Collectors.toList()); + int oneDetectionValueSize = + Integer.valueOf(mapperConfig.getParameterValue(MAPPER_DIMENSION_VALUE_SIZE)); + List dimensionValues = + hanlpMapResults.stream() + .filter(entry -> mapperHelper.existDimensionValues(entry.getNatures())) + .limit(oneDetectionValueSize) + .collect(Collectors.toList()); - Integer oneDetectionSize = Integer.valueOf(mapperConfig.getParameterValue(MAPPER_DETECTION_SIZE)); + Integer oneDetectionSize = + Integer.valueOf(mapperConfig.getParameterValue(MAPPER_DETECTION_SIZE)); List oneRoundResults = new ArrayList<>(); // add the dimensionValue if it exists if (CollectionUtils.isNotEmpty(dimensionValues)) { oneRoundResults.addAll(dimensionValues); } - // fill the rest of the list with other results, excluding the dimensionValue if it was added + // fill the rest of the list with other results, excluding the dimensionValue if it was + // added if (oneRoundResults.size() < oneDetectionSize) { - List additionalResults = hanlpMapResults.stream() - .filter(entry -> !mapperHelper.existDimensionValues(entry.getNatures()) - && !oneRoundResults.contains(entry)) - .limit(oneDetectionSize - oneRoundResults.size()) - .collect(Collectors.toList()); + List additionalResults = + hanlpMapResults.stream() + .filter( + entry -> + !mapperHelper.existDimensionValues(entry.getNatures()) + && !oneRoundResults.contains(entry)) + .limit(oneDetectionSize - oneRoundResults.size()) + .collect(Collectors.toList()); oneRoundResults.addAll(additionalResults); } // step6. select mapResul in one round @@ -128,14 +164,21 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { } public double getThresholdMatch(List natures, ChatQueryContext chatQueryContext) { - Double threshold = Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD)); - Double minThreshold = Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD_MIN)); + Double threshold = + Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD)); + Double minThreshold = + Double.valueOf( + mapperConfig.getParameterValue(MapperConfig.MAPPER_NAME_THRESHOLD_MIN)); if (mapperHelper.existDimensionValues(natures)) { - threshold = Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_VALUE_THRESHOLD)); - minThreshold = Double.valueOf(mapperConfig.getParameterValue(MapperConfig.MAPPER_VALUE_THRESHOLD_MIN)); + threshold = + Double.valueOf( + mapperConfig.getParameterValue(MapperConfig.MAPPER_VALUE_THRESHOLD)); + minThreshold = + Double.valueOf( + mapperConfig.getParameterValue( + MapperConfig.MAPPER_VALUE_THRESHOLD_MIN)); } return getThreshold(threshold, minThreshold, chatQueryContext.getMapModeEnum()); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java index a0abeb9ea..df0bb74d4 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java @@ -22,9 +22,9 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/*** - * A mapper that recognizes schema elements with keyword. - * It leverages two matching strategies: HanlpDictMatchStrategy and DatabaseMatchStrategy. +/** + * * A mapper that recognizes schema elements with keyword. It leverages two matching strategies: + * HanlpDictMatchStrategy and DatabaseMatchStrategy. */ @Slf4j public class KeywordMapper extends BaseMapper { @@ -32,29 +32,40 @@ public class KeywordMapper extends BaseMapper { @Override public void doMap(ChatQueryContext chatQueryContext) { String queryText = chatQueryContext.getQueryText(); - //1.hanlpDict Match - List terms = HanlpHelper.getTerms(queryText, chatQueryContext.getModelIdToDataSetIds()); - HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class); + // 1.hanlpDict Match + List terms = + HanlpHelper.getTerms(queryText, chatQueryContext.getModelIdToDataSetIds()); + HanlpDictMatchStrategy hanlpMatchStrategy = + ContextUtils.getBean(HanlpDictMatchStrategy.class); - List hanlpMapResults = hanlpMatchStrategy.getMatches(chatQueryContext, terms); + List hanlpMapResults = + hanlpMatchStrategy.getMatches(chatQueryContext, terms); convertHanlpMapResultToMapInfo(hanlpMapResults, chatQueryContext, terms); - //2.database Match - DatabaseMatchStrategy databaseMatchStrategy = ContextUtils.getBean(DatabaseMatchStrategy.class); + // 2.database Match + DatabaseMatchStrategy databaseMatchStrategy = + ContextUtils.getBean(DatabaseMatchStrategy.class); - List databaseResults = databaseMatchStrategy.getMatches(chatQueryContext, terms); + List databaseResults = + databaseMatchStrategy.getMatches(chatQueryContext, terms); convertDatabaseMapResultToMapInfo(chatQueryContext, databaseResults); } - private void convertHanlpMapResultToMapInfo(List mapResults, ChatQueryContext chatQueryContext, - List terms) { + private void convertHanlpMapResultToMapInfo( + List mapResults, + ChatQueryContext chatQueryContext, + List terms) { if (CollectionUtils.isEmpty(mapResults)) { return; } HanlpHelper.transLetterOriginal(mapResults); - Map wordNatureToFrequency = terms.stream().collect( - Collectors.toMap(entry -> entry.getWord() + entry.getNature(), - term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2)); + Map wordNatureToFrequency = + terms.stream() + .collect( + Collectors.toMap( + entry -> entry.getWord() + entry.getNature(), + term -> Long.valueOf(term.getFrequency()), + (value1, value2) -> value2)); for (HanlpMapResult hanlpMapResult : mapResults) { for (String nature : hanlpMapResult.getNatures()) { @@ -67,55 +78,70 @@ public class KeywordMapper extends BaseMapper { continue; } Long elementID = NatureHelper.getElementID(nature); - SchemaElement element = getSchemaElement(dataSetId, elementType, - elementID, chatQueryContext.getSemanticSchema()); + SchemaElement element = + getSchemaElement( + dataSetId, + elementType, + elementID, + chatQueryContext.getSemanticSchema()); if (element == null) { continue; } Long frequency = wordNatureToFrequency.get(hanlpMapResult.getName() + nature); - SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() - .element(element) - .frequency(frequency) - .word(hanlpMapResult.getName()) - .similarity(hanlpMapResult.getSimilarity()) - .detectWord(hanlpMapResult.getDetectWord()) - .build(); + SchemaElementMatch schemaElementMatch = + SchemaElementMatch.builder() + .element(element) + .frequency(frequency) + .word(hanlpMapResult.getName()) + .similarity(hanlpMapResult.getSimilarity()) + .detectWord(hanlpMapResult.getDetectWord()) + .build(); addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch); } } } - private void convertDatabaseMapResultToMapInfo(ChatQueryContext chatQueryContext, - List mapResults) { + private void convertDatabaseMapResultToMapInfo( + ChatQueryContext chatQueryContext, List mapResults) { MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); for (DatabaseMapResult match : mapResults) { SchemaElement schemaElement = match.getSchemaElement(); - Set regElementSet = getRegElementSet(chatQueryContext.getMapInfo(), schemaElement); + Set regElementSet = + getRegElementSet(chatQueryContext.getMapInfo(), schemaElement); if (regElementSet.contains(schemaElement.getId())) { continue; } - SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() - .element(schemaElement) - .word(schemaElement.getName()) - .detectWord(match.getDetectWord()) - .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) - .similarity(mapperHelper.getSimilarity(match.getDetectWord(), schemaElement.getName())) - .build(); + SchemaElementMatch schemaElementMatch = + SchemaElementMatch.builder() + .element(schemaElement) + .word(schemaElement.getName()) + .detectWord(match.getDetectWord()) + .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) + .similarity( + mapperHelper.getSimilarity( + match.getDetectWord(), schemaElement.getName())) + .build(); log.info("add to schema, elementMatch {}", schemaElementMatch); - addToSchemaMap(chatQueryContext.getMapInfo(), schemaElement.getDataSetId(), schemaElementMatch); + addToSchemaMap( + chatQueryContext.getMapInfo(), + schemaElement.getDataSetId(), + schemaElementMatch); } } private Set getRegElementSet(SchemaMapInfo schemaMap, SchemaElement schemaElement) { - List elements = schemaMap.getMatchedElements(schemaElement.getDataSetId()); + List elements = + schemaMap.getMatchedElements(schemaElement.getDataSetId()); if (CollectionUtils.isEmpty(elements)) { return new HashSet<>(); } return elements.stream() - .filter(elementMatch -> - SchemaElementType.METRIC.equals(elementMatch.getElement().getType()) - || SchemaElementType.DIMENSION.equals(elementMatch.getElement().getType())) + .filter( + elementMatch -> + SchemaElementType.METRIC.equals(elementMatch.getElement().getType()) + || SchemaElementType.DIMENSION.equals( + elementMatch.getElement().getType())) .map(elementMatch -> elementMatch.getElement().getId()) .collect(Collectors.toSet()); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java index 62a0458bd..2786fb6a1 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java @@ -8,87 +8,128 @@ import org.springframework.stereotype.Service; public class MapperConfig extends ParameterConfig { public static final Parameter MAPPER_DETECTION_SIZE = - new Parameter("s2.mapper.detection.size", "8", + new Parameter( + "s2.mapper.detection.size", + "8", "一次探测返回结果个数", "在每次探测后, 将前后缀匹配的结果合并, 并根据相似度阈值过滤后的结果个数", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter MAPPER_DETECTION_MAX_SIZE = - new Parameter("s2.mapper.detection.max.size", "20", + new Parameter( + "s2.mapper.detection.max.size", + "20", "一次探测前后缀匹配结果返回个数", "单次前后缀匹配返回的结果个数", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter MAPPER_NAME_THRESHOLD = - new Parameter("s2.mapper.name.threshold", "0.3", + new Parameter( + "s2.mapper.name.threshold", + "0.3", "指标名、维度名文本相似度阈值", "文本片段和匹配到的指标、维度名计算出来的编辑距离阈值, 若超出该阈值, 则舍弃", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter MAPPER_NAME_THRESHOLD_MIN = - new Parameter("s2.mapper.name.min.threshold", "0.25", + new Parameter( + "s2.mapper.name.min.threshold", + "0.25", "指标名、维度名最小文本相似度阈值", "指标名、维度名相似度阈值在动态调整中的最低值", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter MAPPER_DIMENSION_VALUE_SIZE = - new Parameter("s2.mapper.value.size", "1", + new Parameter( + "s2.mapper.value.size", + "1", "一次探测返回维度值结果个数", "在每次探测后, 将前后缀匹配的结果合并, 并根据相似度阈值过滤后的维度值结果个数", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter MAPPER_VALUE_THRESHOLD = - new Parameter("s2.mapper.value.threshold", "0.5", + new Parameter( + "s2.mapper.value.threshold", + "0.5", "维度值文本相似度阈值", "文本片段和匹配到的维度值计算出来的编辑距离阈值, 若超出该阈值, 则舍弃", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter MAPPER_VALUE_THRESHOLD_MIN = - new Parameter("s2.mapper.value.min.threshold", "0.3", + new Parameter( + "s2.mapper.value.min.threshold", + "0.3", "维度值最小文本相似度阈值", "维度值相似度阈值在动态调整中的最低值", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_TEXT_SIZE = - new Parameter("s2.mapper.embedding.word.size", "4", + new Parameter( + "s2.mapper.embedding.word.size", + "4", "用于向量召回文本长度", "为提高向量召回效率, 按指定长度进行向量语义召回", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_TEXT_STEP = - new Parameter("s2.mapper.embedding.word.step", "3", + new Parameter( + "s2.mapper.embedding.word.step", + "3", "向量召回文本每步长度", "为提高向量召回效率, 按指定每步长度进行召回", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_BATCH = - new Parameter("s2.mapper.embedding.batch", "50", + new Parameter( + "s2.mapper.embedding.batch", + "50", "批量向量召回文本请求个数", "每次进行向量语义召回的原始文本片段个数", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_NUMBER = - new Parameter("s2.mapper.embedding.number", "5", + new Parameter( + "s2.mapper.embedding.number", + "5", "批量向量召回文本返回结果个数", "每个文本进行向量语义召回的文本结果个数", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_THRESHOLD = - new Parameter("s2.mapper.embedding.threshold", "0.98", + new Parameter( + "s2.mapper.embedding.threshold", + "0.98", "向量召回相似度阈值", "相似度小于该阈值的则舍弃", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_THRESHOLD_MIN = - new Parameter("s2.mapper.embedding.min.threshold", "0.9", + new Parameter( + "s2.mapper.embedding.min.threshold", + "0.9", "向量召回最小相似度阈值", "向量召回相似度阈值在动态调整中的最低值", - "number", "Mapper相关配置"); + "number", + "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_ROUND_NUMBER = - new Parameter("s2.mapper.embedding.round.number", "10", + new Parameter( + "s2.mapper.embedding.round.number", + "10", "向量召回最小相似度阈值", "向量召回相似度阈值在动态调整中的最低值", - "number", "Mapper相关配置"); - + "number", + "Mapper相关配置"); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperHelper.java index f481056db..fe6373e12 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperHelper.java @@ -29,8 +29,11 @@ public class MapperHelper { } public Integer getStepOffset(List termList, Integer index) { - List offsetList = termList.stream().sorted(Comparator.comparing(S2Term::getOffset)) - .map(term -> term.getOffset()).collect(Collectors.toList()); + List offsetList = + termList.stream() + .sorted(Comparator.comparing(S2Term::getOffset)) + .map(term -> term.getOffset()) + .collect(Collectors.toList()); for (int j = 0; j < termList.size() - 1; j++) { if (offsetList.get(j) <= index && offsetList.get(j + 1) > index) { @@ -40,8 +43,9 @@ public class MapperHelper { return index; } - /*** - * exist dimension values + /** + * * exist dimension values + * * @param natures * @return */ @@ -63,8 +67,9 @@ public class MapperHelper { return false; } - /*** - * get similarity + /** + * * get similarity + * * @param detectSegment * @param matchName * @return @@ -72,7 +77,8 @@ public class MapperHelper { public double getSimilarity(String detectSegment, String matchName) { String detectSegmentLower = detectSegment == null ? null : detectSegment.toLowerCase(); String matchNameLower = matchName == null ? null : matchName.toLowerCase(); - return 1 - (double) EditDistance.compute(detectSegmentLower, matchNameLower) / Math.max(matchName.length(), - detectSegment.length()); + return 1 + - (double) EditDistance.compute(detectSegmentLower, matchNameLower) + / Math.max(matchName.length(), detectSegment.length()); } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchStrategy.java index 96289798b..af0bf9ed7 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchStrategy.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.mapper; - import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.chat.ChatQueryContext; @@ -9,11 +8,10 @@ import java.util.Map; import java.util.Set; /** - * MatchStrategy encapsulates a concrete matching algorithm - * executed during query or search process. + * MatchStrategy encapsulates a concrete matching algorithm executed during query or search process. */ public interface MatchStrategy { - Map> match(ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds); - -} \ No newline at end of file + Map> match( + ChatQueryContext chatQueryContext, List terms, Set detectDataSetIds); +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchText.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchText.java index 8313feae3..a3482ba22 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchText.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MatchText.java @@ -24,7 +24,8 @@ public class MatchText { return false; } MatchText that = (MatchText) o; - return Objects.equals(regText, that.regText) && Objects.equals(detectSegment, that.detectSegment); + return Objects.equals(regText, that.regText) + && Objects.equals(detectSegment, that.detectSegment); } @Override diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/ModelWithSemanticType.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/ModelWithSemanticType.java index 072b0a39b..96b8856b0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/ModelWithSemanticType.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/ModelWithSemanticType.java @@ -17,4 +17,4 @@ public class ModelWithSemanticType implements Serializable { this.model = model; this.schemaElementType = schemaElementType; } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/QueryFilterMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/QueryFilterMapper.java index 7499690da..da75b37d9 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/QueryFilterMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/QueryFilterMapper.java @@ -32,7 +32,8 @@ public class QueryFilterMapper extends BaseMapper { SchemaMapInfo schemaMapInfo = chatQueryContext.getMapInfo(); clearOtherSchemaElementMatch(dataSetIds, schemaMapInfo); for (Long dataSetId : dataSetIds) { - List schemaElementMatches = schemaMapInfo.getMatchedElements(dataSetId); + List schemaElementMatches = + schemaMapInfo.getMatchedElements(dataSetId); if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); schemaMapInfo.setMatchedElements(dataSetId, schemaElementMatches); @@ -42,14 +43,17 @@ public class QueryFilterMapper extends BaseMapper { } private void clearOtherSchemaElementMatch(Set viewIds, SchemaMapInfo schemaMapInfo) { - for (Map.Entry> entry : schemaMapInfo.getDataSetElementMatches().entrySet()) { + for (Map.Entry> entry : + schemaMapInfo.getDataSetElementMatches().entrySet()) { if (!viewIds.contains(entry.getKey())) { entry.getValue().clear(); } } } - private void addValueSchemaElementMatch(Long dataSetId, ChatQueryContext chatQueryContext, + private void addValueSchemaElementMatch( + Long dataSetId, + ChatQueryContext chatQueryContext, List candidateElementMatches) { QueryFilters queryFilters = chatQueryContext.getQueryFilters(); if (queryFilters == null || CollectionUtils.isEmpty(queryFilters.getFilters())) { @@ -59,33 +63,41 @@ public class QueryFilterMapper extends BaseMapper { if (checkExistSameValueSchemaElementMatch(filter, candidateElementMatches)) { continue; } - SchemaElement element = SchemaElement.builder() - .id(filter.getElementID()) - .name(String.valueOf(filter.getValue())) - .type(SchemaElementType.VALUE) - .bizName(filter.getBizName()) - .dataSetId(dataSetId) - .build(); - SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() - .element(element) - .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) - .word(String.valueOf(filter.getValue())) - .similarity(similarity) - .detectWord(Constants.EMPTY) - .build(); + SchemaElement element = + SchemaElement.builder() + .id(filter.getElementID()) + .name(String.valueOf(filter.getValue())) + .type(SchemaElementType.VALUE) + .bizName(filter.getBizName()) + .dataSetId(dataSetId) + .build(); + SchemaElementMatch schemaElementMatch = + SchemaElementMatch.builder() + .element(element) + .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) + .word(String.valueOf(filter.getValue())) + .similarity(similarity) + .detectWord(Constants.EMPTY) + .build(); candidateElementMatches.add(schemaElementMatch); } chatQueryContext.getMapInfo().setMatchedElements(dataSetId, candidateElementMatches); } - private boolean checkExistSameValueSchemaElementMatch(QueryFilter queryFilter, - List schemaElementMatches) { - List valueSchemaElements = schemaElementMatches.stream().filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); + private boolean checkExistSameValueSchemaElementMatch( + QueryFilter queryFilter, List schemaElementMatches) { + List valueSchemaElements = + schemaElementMatches.stream() + .filter( + schemaElementMatch -> + SchemaElementType.VALUE.equals( + schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); for (SchemaElementMatch schemaElementMatch : valueSchemaElements) { if (schemaElementMatch.getElement().getId().equals(queryFilter.getElementID()) - && schemaElementMatch.getWord().equals(String.valueOf(queryFilter.getValue()))) { + && schemaElementMatch + .getWord() + .equals(String.valueOf(queryFilter.getValue()))) { return true; } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SchemaMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SchemaMapper.java index 15ee54723..a64346d2e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SchemaMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SchemaMapper.java @@ -1,11 +1,10 @@ package com.tencent.supersonic.headless.chat.mapper; - import com.tencent.supersonic.headless.chat.ChatQueryContext; /** - * A schema mapper identifies references to schema elements(metrics/dimensions/entities/values) - * in user queries. It matches the query text against the knowledge base. + * A schema mapper identifies references to schema elements(metrics/dimensions/entities/values) in + * user queries. It matches the query text against the knowledge base. */ public interface SchemaMapper { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SearchMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SearchMatchStrategy.java index 2bcea8b29..9a67ad75d 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SearchMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/SearchMatchStrategy.java @@ -20,20 +20,18 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** - * SearchMatchStrategy encapsulates a concrete matching algorithm - * executed during search process. + * SearchMatchStrategy encapsulates a concrete matching algorithm executed during search process. */ @Service public class SearchMatchStrategy extends BaseMatchStrategy { private static final int SEARCH_SIZE = 3; - @Autowired - private KnowledgeBaseService knowledgeBaseService; + @Autowired private KnowledgeBaseService knowledgeBaseService; @Override - public Map> match(ChatQueryContext chatQueryContext, List originals, - Set detectDataSetIds) { + public Map> match( + ChatQueryContext chatQueryContext, List originals, Set detectDataSetIds) { String text = chatQueryContext.getQueryText(); Map regOffsetToLength = getRegOffsetToLength(originals); @@ -52,39 +50,58 @@ public class SearchMatchStrategy extends BaseMatchStrategy { } } Map> regTextMap = new ConcurrentHashMap<>(); - detectIndexList.stream().parallel().forEach(detectIndex -> { - String regText = text.substring(0, detectIndex); - String detectSegment = text.substring(detectIndex); + detectIndexList.stream() + .parallel() + .forEach( + detectIndex -> { + String regText = text.substring(0, detectIndex); + String detectSegment = text.substring(detectIndex); - if (StringUtils.isNotEmpty(detectSegment)) { - List hanlpMapResults = knowledgeBaseService.prefixSearch(detectSegment, - SearchService.SEARCH_SIZE, - chatQueryContext.getModelIdToDataSetIds(), - detectDataSetIds); - List suffixHanlpMapResults = knowledgeBaseService.suffixSearch( - detectSegment, - SEARCH_SIZE, - chatQueryContext.getModelIdToDataSetIds(), - detectDataSetIds); - hanlpMapResults.addAll(suffixHanlpMapResults); - // remove entity name where search - hanlpMapResults = hanlpMapResults.stream().filter(entry -> { - List natures = entry.getNatures().stream() - .filter(nature -> !nature.endsWith(DictWordType.ENTITY.getType())) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(natures)) { - return false; + if (StringUtils.isNotEmpty(detectSegment)) { + List hanlpMapResults = + knowledgeBaseService.prefixSearch( + detectSegment, + SearchService.SEARCH_SIZE, + chatQueryContext.getModelIdToDataSetIds(), + detectDataSetIds); + List suffixHanlpMapResults = + knowledgeBaseService.suffixSearch( + detectSegment, + SEARCH_SIZE, + chatQueryContext.getModelIdToDataSetIds(), + detectDataSetIds); + hanlpMapResults.addAll(suffixHanlpMapResults); + // remove entity name where search + hanlpMapResults = + hanlpMapResults.stream() + .filter( + entry -> { + List natures = + entry.getNatures().stream() + .filter( + nature -> + !nature + .endsWith( + DictWordType + .ENTITY + .getType())) + .collect( + Collectors + .toList()); + if (CollectionUtils.isEmpty(natures)) { + return false; + } + return true; + }) + .collect(Collectors.toList()); + MatchText matchText = + MatchText.builder() + .regText(regText) + .detectSegment(detectSegment) + .build(); + regTextMap.put(matchText, hanlpMapResults); } - return true; - }).collect(Collectors.toList()); - MatchText matchText = MatchText.builder() - .regText(regText) - .detectSegment(detectSegment) - .build(); - regTextMap.put(matchText, hanlpMapResults); - } - } - ); + }); return regTextMap; } @@ -99,9 +116,10 @@ public class SearchMatchStrategy extends BaseMatchStrategy { } @Override - public void detectByStep(ChatQueryContext chatQueryContext, Set existResults, - Set detectDataSetIds, String detectSegment, int offset) { - - } - + public void detectByStep( + ChatQueryContext chatQueryContext, + Set existResults, + Set detectDataSetIds, + String detectSegment, + int offset) {} } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TermDescMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TermDescMapper.java index 6abb739b1..49488e8d2 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TermDescMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TermDescMapper.java @@ -5,17 +5,17 @@ import com.tencent.supersonic.headless.chat.ChatQueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; + import java.util.List; -/*** - * A mapper that map the description of the term. - */ +/** * A mapper that map the description of the term. */ @Slf4j public class TermDescMapper extends BaseMapper { @Override public void doMap(ChatQueryContext chatQueryContext) { - List termDescriptionToMap = chatQueryContext.getMapInfo().getTermDescriptionToMap(); + List termDescriptionToMap = + chatQueryContext.getMapInfo().getTermDescriptionToMap(); if (CollectionUtils.isEmpty(termDescriptionToMap)) { return; } @@ -37,5 +37,4 @@ public class TermDescMapper extends BaseMapper { chatQueryContext.setQueryText(chatQueryContext.getOriQueryText()); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java index 5e02725bf..8c4ccfe42 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java @@ -13,53 +13,82 @@ import java.util.List; public class ParserConfig extends ParameterConfig { public static final Parameter PARSER_STRATEGY_TYPE = - new Parameter("s2.parser.s2sql.strategy", "ONE_PASS_SELF_CONSISTENCY", + new Parameter( + "s2.parser.s2sql.strategy", + "ONE_PASS_SELF_CONSISTENCY", "LLM解析生成S2SQL策略", "ONE_PASS_SELF_CONSISTENCY: 通过投票方式一步生成sql", - "list", "Parser相关配置", + "list", + "Parser相关配置", Lists.newArrayList("ONE_PASS_SELF_CONSISTENCY")); public static final Parameter PARSER_LINKING_VALUE_ENABLE = - new Parameter("s2.parser.linking.value.enable", "true", - "是否将Mapper探测识别到的维度值提供给大模型", "为了数据安全考虑, 这里可进行开关选择", - "bool", "Parser相关配置"); + new Parameter( + "s2.parser.linking.value.enable", + "true", + "是否将Mapper探测识别到的维度值提供给大模型", + "为了数据安全考虑, 这里可进行开关选择", + "bool", + "Parser相关配置"); public static final Parameter PARSER_TEXT_LENGTH_THRESHOLD = - new Parameter("s2.parser.text.length.threshold", "10", - "用户输入文本长短阈值", "文本超过该阈值为长文本", - "number", "Parser相关配置"); + new Parameter( + "s2.parser.text.length.threshold", + "10", + "用户输入文本长短阈值", + "文本超过该阈值为长文本", + "number", + "Parser相关配置"); public static final Parameter PARSER_TEXT_LENGTH_THRESHOLD_SHORT = - new Parameter("s2.parser.text.threshold.short", "0.5", + new Parameter( + "s2.parser.text.threshold.short", + "0.5", "短文本匹配阈值", "由于请求大模型耗时较长, 因此如果有规则类型的Query得分达到阈值,则跳过大模型的调用," - + "\n如果是短文本, 若query得分/文本长度>该阈值, 则跳过当前parser", - "number", "Parser相关配置"); + + "\n如果是短文本, 若query得分/文本长度>该阈值, 则跳过当前parser", + "number", + "Parser相关配置"); public static final Parameter PARSER_TEXT_LENGTH_THRESHOLD_LONG = - new Parameter("s2.parser.text.threshold.long", "0.8", - "长文本匹配阈值", "如果是长文本, 若query得分/文本长度>该阈值, 则跳过当前parser", - "number", "Parser相关配置"); + new Parameter( + "s2.parser.text.threshold.long", + "0.8", + "长文本匹配阈值", + "如果是长文本, 若query得分/文本长度>该阈值, 则跳过当前parser", + "number", + "Parser相关配置"); public static final Parameter PARSER_EXEMPLAR_RECALL_NUMBER = - new Parameter("s2.parser.exemplar-recall.number", "10", - "exemplar召回个数", "", - "number", "Parser相关配置"); + new Parameter( + "s2.parser.exemplar-recall.number", + "10", + "exemplar召回个数", + "", + "number", + "Parser相关配置"); public static final Parameter PARSER_FEW_SHOT_NUMBER = - new Parameter("s2.parser.few-shot.number", "3", - "few-shot样例个数", "样例越多效果可能越好，但token消耗越大", - "number", "Parser相关配置"); + new Parameter( + "s2.parser.few-shot.number", + "3", + "few-shot样例个数", + "样例越多效果可能越好，但token消耗越大", + "number", + "Parser相关配置"); public static final Parameter PARSER_SELF_CONSISTENCY_NUMBER = - new Parameter("s2.parser.self-consistency.number", "1", - "self-consistency执行个数", "执行越多效果可能越好，但token消耗越大", - "number", "Parser相关配置"); + new Parameter( + "s2.parser.self-consistency.number", + "1", + "self-consistency执行个数", + "执行越多效果可能越好，但token消耗越大", + "number", + "Parser相关配置"); public static final Parameter PARSER_SHOW_COUNT = - new Parameter("s2.parser.show.count", "3", - "解析结果展示个数", "前端展示的解析个数", - "number", "Parser相关配置"); + new Parameter( + "s2.parser.show.count", "3", "解析结果展示个数", "前端展示的解析个数", "number", "Parser相关配置"); @Override public List getSysParameters() { @@ -67,8 +96,6 @@ public class ParserConfig extends ParameterConfig { PARSER_LINKING_VALUE_ENABLE, PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, - PARSER_SHOW_COUNT - ); + PARSER_SHOW_COUNT); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java index 92e8a15d8..4c749645c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java @@ -1,18 +1,18 @@ package com.tencent.supersonic.headless.chat.parser; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.SemanticQuery; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.headless.chat.ChatQueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -22,9 +22,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/** - * QueryTypeParser resolves query type as either METRIC or TAG, or ID. - */ +/** QueryTypeParser resolves query type as either METRIC or TAG, or ID. */ @Slf4j public class QueryTypeParser implements SemanticParser { @@ -37,8 +35,8 @@ public class QueryTypeParser implements SemanticParser { for (SemanticQuery semanticQuery : candidateQueries) { // 1.init S2SQL Long dataSetId = semanticQuery.getParseInfo().getDataSetId(); - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema() - .getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); semanticQuery.initS2Sql(dataSetSchema, user); // 2.set queryType QueryType queryType = getQueryType(chatQueryContext, semanticQuery); @@ -53,23 +51,25 @@ public class QueryTypeParser implements SemanticParser { return QueryType.DETAIL; } - //1. entity queryType + // 1. entity queryType Long dataSetId = parseInfo.getDataSetId(); SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) { List whereFields = SqlSelectHelper.getWhereFields(sqlInfo.getParsedS2SQL()); List whereFilterByTimeFields = filterByTimeFields(whereFields); if (CollectionUtils.isNotEmpty(whereFilterByTimeFields)) { - Set ids = semanticSchema.getEntities(dataSetId).stream().map(SchemaElement::getName) - .collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(ids) && ids.stream() - .anyMatch(whereFilterByTimeFields::contains)) { + Set ids = + semanticSchema.getEntities(dataSetId).stream() + .map(SchemaElement::getName) + .collect(Collectors.toSet()); + if (CollectionUtils.isNotEmpty(ids) + && ids.stream().anyMatch(whereFilterByTimeFields::contains)) { return QueryType.ID; } } } - //2. metric queryType + // 2. metric queryType if (selectContainsMetric(sqlInfo, dataSetId, semanticSchema)) { return QueryType.METRIC; } @@ -78,20 +78,22 @@ public class QueryTypeParser implements SemanticParser { } private static List filterByTimeFields(List whereFields) { - List selectAndWhereFilterByTimeFields = whereFields - .stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) - .collect(Collectors.toList()); + List selectAndWhereFilterByTimeFields = + whereFields.stream() + .filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) + .collect(Collectors.toList()); return selectAndWhereFilterByTimeFields; } - private static boolean selectContainsMetric(SqlInfo sqlInfo, Long dataSetId, SemanticSchema semanticSchema) { + private static boolean selectContainsMetric( + SqlInfo sqlInfo, Long dataSetId, SemanticSchema semanticSchema) { List selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getParsedS2SQL()); List metrics = semanticSchema.getMetrics(dataSetId); if (CollectionUtils.isNotEmpty(metrics)) { - Set metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); + Set metricNameSet = + metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); return selectFields.stream().anyMatch(metricNameSet::contains); } return false; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SatisfactionChecker.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SatisfactionChecker.java index a945eaf3d..e6ff0d05b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SatisfactionChecker.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SatisfactionChecker.java @@ -11,11 +11,9 @@ import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_TE import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_TEXT_LENGTH_THRESHOLD_LONG; import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_TEXT_LENGTH_THRESHOLD_SHORT; - /** - * This checker can be used by semantic parsers to check if query intent - * has already been satisfied by current candidate queries. If so, current - * parser could be skipped. + * This checker can be used by semantic parsers to check if query intent has already been satisfied + * by current candidate queries. If so, current parser could be skipped. */ @Slf4j public class SatisfactionChecker { @@ -51,9 +49,11 @@ public class SatisfactionChecker { } else if (degree < shortTextLengthThreshold) { return false; } - log.info("queryMode:{}, degree:{}, parse info:{}", - semanticParseInfo.getQueryMode(), degree, semanticParseInfo); + log.info( + "queryMode:{}, degree:{}, parse info:{}", + semanticParseInfo.getQueryMode(), + degree, + semanticParseInfo); return true; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SemanticParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SemanticParser.java index ee5cd7632..2d12f64d7 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SemanticParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/SemanticParser.java @@ -3,9 +3,9 @@ package com.tencent.supersonic.headless.chat.parser; import com.tencent.supersonic.headless.chat.ChatQueryContext; /** - * A semantic parser understands user queries and generates semantic query statement. - * SuperSonic leverages a combination of rule-based and LLM-based parsers, - * each of which deals with specific scenarios. + * A semantic parser understands user queries and generates semantic query statement. SuperSonic + * leverages a combination of rule-based and LLM-based parsers, each of which deals with specific + * scenarios. */ public interface SemanticParser { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/DataSetResolver.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/DataSetResolver.java index fe044a671..ef2df1c32 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/DataSetResolver.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/DataSetResolver.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.parser.llm; - import com.tencent.supersonic.headless.chat.ChatQueryContext; import java.util.Set; @@ -8,5 +7,4 @@ import java.util.Set; public interface DataSetResolver { Long resolve(ChatQueryContext chatQueryContext, Set restrictiveModels); - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java index 96d98e6ac..12d518d95 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java @@ -3,10 +3,11 @@ package com.tencent.supersonic.headless.chat.parser.llm; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.headless.chat.query.SemanticQuery; import com.tencent.supersonic.headless.chat.ChatQueryContext; +import com.tencent.supersonic.headless.chat.query.SemanticQuery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -21,9 +22,9 @@ import java.util.stream.Collectors; @Slf4j public class HeuristicDataSetResolver implements DataSetResolver { - protected static Long selectDataSetBySchemaElementMatchScore(Map dataSetQueryModes, - SchemaMapInfo schemaMap) { - //dataSet count priority + protected static Long selectDataSetBySchemaElementMatchScore( + Map dataSetQueryModes, SchemaMapInfo schemaMap) { + // dataSet count priority Long dataSetIdByDataSetCount = getDataSetIdByMatchDataSetScore(schemaMap); if (Objects.nonNull(dataSetIdByDataSetCount)) { log.info("selectDataSet by dataSet count:{}", dataSetIdByDataSetCount); @@ -38,16 +39,24 @@ public class HeuristicDataSetResolver implements DataSetResolver { return dataSetSelect; } } else { - Entry maxDataSet = dataSetTypeMap.entrySet().stream() - .filter(entry -> dataSetQueryModes.containsKey(entry.getKey())) - .sorted((o1, o2) -> { - int difference = o2.getValue().getCount() - o1.getValue().getCount(); - if (difference == 0) { - return (int) ((o2.getValue().getMaxSimilarity() - - o1.getValue().getMaxSimilarity()) * 100); - } - return difference; - }).findFirst().orElse(null); + Entry maxDataSet = + dataSetTypeMap.entrySet().stream() + .filter(entry -> dataSetQueryModes.containsKey(entry.getKey())) + .sorted( + (o1, o2) -> { + int difference = + o2.getValue().getCount() - o1.getValue().getCount(); + if (difference == 0) { + return (int) + ((o2.getValue().getMaxSimilarity() + - o1.getValue() + .getMaxSimilarity()) + * 100); + } + return difference; + }) + .findFirst() + .orElse(null); if (maxDataSet != null) { log.info("selectDataSet with multiple DataSets [{}]", maxDataSet.getKey()); return maxDataSet.getKey(); @@ -57,26 +66,40 @@ public class HeuristicDataSetResolver implements DataSetResolver { } private static Long getDataSetIdByMatchDataSetScore(SchemaMapInfo schemaMap) { - Map> dataSetElementMatches = schemaMap.getDataSetElementMatches(); - // calculate dataSet match score, matched element gets 1.0 point, and inherit element gets 0.5 point + Map> dataSetElementMatches = + schemaMap.getDataSetElementMatches(); + // calculate dataSet match score, matched element gets 1.0 point, and inherit element gets + // 0.5 point Map dataSetIdToDataSetScore = new HashMap<>(); if (Objects.nonNull(dataSetElementMatches)) { - for (Entry> dataSetElementMatch : dataSetElementMatches.entrySet()) { + for (Entry> dataSetElementMatch : + dataSetElementMatches.entrySet()) { Long dataSetId = dataSetElementMatch.getKey(); - List dataSetMatchesScore = dataSetElementMatch.getValue().stream() - .filter(elementMatch -> elementMatch.getSimilarity() >= 1) - .filter(elementMatch -> SchemaElementType.DATASET.equals(elementMatch.getElement().getType())) - .map(elementMatch -> elementMatch.isInherited() ? 0.5 : 1.0).collect(Collectors.toList()); + List dataSetMatchesScore = + dataSetElementMatch.getValue().stream() + .filter(elementMatch -> elementMatch.getSimilarity() >= 1) + .filter( + elementMatch -> + SchemaElementType.DATASET.equals( + elementMatch.getElement().getType())) + .map(elementMatch -> elementMatch.isInherited() ? 0.5 : 1.0) + .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(dataSetMatchesScore)) { // get sum of dataSet match score - double score = dataSetMatchesScore.stream().mapToDouble(Double::doubleValue).sum(); + double score = + dataSetMatchesScore.stream().mapToDouble(Double::doubleValue).sum(); dataSetIdToDataSetScore.put(dataSetId, score); } } - Entry maxDataSetScore = dataSetIdToDataSetScore.entrySet().stream() - .max(Comparator.comparingDouble(Entry::getValue)).orElse(null); - log.info("maxDataSetCount:{},dataSetIdToDataSetCount:{}", maxDataSetScore, dataSetIdToDataSetScore); + Entry maxDataSetScore = + dataSetIdToDataSetScore.entrySet().stream() + .max(Comparator.comparingDouble(Entry::getValue)) + .orElse(null); + log.info( + "maxDataSetCount:{},dataSetIdToDataSetCount:{}", + maxDataSetScore, + dataSetIdToDataSetScore); if (Objects.nonNull(maxDataSetScore)) { return maxDataSetScore.getKey(); } @@ -86,8 +109,10 @@ public class HeuristicDataSetResolver implements DataSetResolver { public static Map getDataSetTypeMap(SchemaMapInfo schemaMap) { Map dataSetCount = new HashMap<>(); - for (Entry> entry : schemaMap.getDataSetElementMatches().entrySet()) { - List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); + for (Entry> entry : + schemaMap.getDataSetElementMatches().entrySet()) { + List schemaElementMatches = + schemaMap.getMatchedElements(entry.getKey()); if (schemaElementMatches != null && schemaElementMatches.size() > 0) { if (!dataSetCount.containsKey(entry.getKey())) { dataSetCount.put(entry.getKey(), new DataSetMatchResult()); @@ -95,17 +120,23 @@ public class HeuristicDataSetResolver implements DataSetResolver { DataSetMatchResult dataSetMatchResult = dataSetCount.get(entry.getKey()); Set schemaElementTypes = new HashSet<>(); schemaElementMatches.stream() - .forEach(schemaElementMatch -> schemaElementTypes.add( - schemaElementMatch.getElement().getType())); - SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() - .sorted((o1, o2) -> - ((int) ((o2.getSimilarity() - o1.getSimilarity()) * 100)) - ).findFirst().orElse(null); + .forEach( + schemaElementMatch -> + schemaElementTypes.add( + schemaElementMatch.getElement().getType())); + SchemaElementMatch schemaElementMatchMax = + schemaElementMatches.stream() + .sorted( + (o1, o2) -> + ((int) + ((o2.getSimilarity() - o1.getSimilarity()) + * 100))) + .findFirst() + .orElse(null); if (schemaElementMatchMax != null) { dataSetMatchResult.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); } dataSetMatchResult.setCount(schemaElementTypes.size()); - } } return dataSetCount; @@ -126,5 +157,4 @@ public class HeuristicDataSetResolver implements DataSetResolver { } return selectDataSetBySchemaElementMatchScore(dataSetQueryModes, mapInfo); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMParserConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMParserConfig.java index 536b53dcf..f1cfe0a35 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMParserConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMParserConfig.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.parser.llm; - import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java index f2704d6ed..a8fdb6feb 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java @@ -1,8 +1,5 @@ package com.tencent.supersonic.headless.chat.parser.llm; -import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_LINKING_VALUE_ENABLE; -import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_STRATEGY_TYPE; - import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; @@ -16,6 +13,13 @@ import com.tencent.supersonic.headless.chat.parser.SatisfactionChecker; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.headless.chat.utils.ComponentFactory; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -25,19 +29,15 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_LINKING_VALUE_ENABLE; +import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_STRATEGY_TYPE; @Slf4j @Service public class LLMRequestService { - @Autowired - private ParserConfig parserConfig; + @Autowired private ParserConfig parserConfig; public boolean isSkip(ChatQueryContext queryCtx) { if (!queryCtx.getText2SQLType().enableLLM()) { @@ -79,7 +79,8 @@ public class LLMRequestService { llmReq.setPriorExts(priorKnowledge); List linking = new ArrayList<>(); - boolean linkingValueEnabled = Boolean.valueOf(parserConfig.getParameterValue(PARSER_LINKING_VALUE_ENABLE)); + boolean linkingValueEnabled = + Boolean.valueOf(parserConfig.getParameterValue(PARSER_LINKING_VALUE_ENABLE)); if (linkingValueEnabled) { linking.addAll(linkingValues); @@ -87,7 +88,8 @@ public class LLMRequestService { llmReq.setLinking(linking); llmReq.setCurrentDate(DateUtils.getBeforeDate(0)); - llmReq.setSqlGenType(LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE))); + llmReq.setSqlGenType( + LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE))); llmReq.setModelConfig(queryCtx.getModelConfig()); llmReq.setPromptConfig(queryCtx.getPromptConfig()); llmReq.setDynamicExemplars(queryCtx.getDynamicExemplars()); @@ -105,21 +107,27 @@ public class LLMRequestService { } protected List getTerms(ChatQueryContext queryCtx, Long dataSetId) { - List matchedElements = queryCtx.getMapInfo().getMatchedElements(dataSetId); + List matchedElements = + queryCtx.getMapInfo().getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(matchedElements)) { return new ArrayList<>(); } return matchedElements.stream() - .filter(schemaElementMatch -> { - SchemaElementType elementType = schemaElementMatch.getElement().getType(); - return SchemaElementType.TERM.equals(elementType); - }).map(schemaElementMatch -> { - LLMReq.Term term = new LLMReq.Term(); - term.setName(schemaElementMatch.getElement().getName()); - term.setDescription(schemaElementMatch.getElement().getDescription()); - term.setAlias(schemaElementMatch.getElement().getAlias()); - return term; - }).collect(Collectors.toList()); + .filter( + schemaElementMatch -> { + SchemaElementType elementType = + schemaElementMatch.getElement().getType(); + return SchemaElementType.TERM.equals(elementType); + }) + .map( + schemaElementMatch -> { + LLMReq.Term term = new LLMReq.Term(); + term.setName(schemaElementMatch.getElement().getName()); + term.setDescription(schemaElementMatch.getElement().getDescription()); + term.setAlias(schemaElementMatch.getElement().getAlias()); + return term; + }) + .collect(Collectors.toList()); } private String getPriorKnowledge(ChatQueryContext queryContext, LLMReq.LLMSchema llmSchema) { @@ -137,23 +145,33 @@ public class LLMRequestService { private Map getFieldNameToDataFormatTypeMap(SemanticSchema semanticSchema) { return semanticSchema.getMetrics().stream() .filter(metric -> Objects.nonNull(metric.getDataFormatType())) - .flatMap(metric -> { - Set> fieldFormatPairs = new HashSet<>(); - String dataFormatType = metric.getDataFormatType(); - fieldFormatPairs.add(Pair.of(metric.getName(), dataFormatType)); - List aliasList = metric.getAlias(); - if (!CollectionUtils.isEmpty(aliasList)) { - aliasList.forEach(alias -> fieldFormatPairs.add(Pair.of(alias, dataFormatType))); - } - return fieldFormatPairs.stream(); - }) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (existing, replacement) -> existing)); + .flatMap( + metric -> { + Set> fieldFormatPairs = new HashSet<>(); + String dataFormatType = metric.getDataFormatType(); + fieldFormatPairs.add(Pair.of(metric.getName(), dataFormatType)); + List aliasList = metric.getAlias(); + if (!CollectionUtils.isEmpty(aliasList)) { + aliasList.forEach( + alias -> + fieldFormatPairs.add( + Pair.of(alias, dataFormatType))); + } + return fieldFormatPairs.stream(); + }) + .collect( + Collectors.toMap( + Pair::getLeft, + Pair::getRight, + (existing, replacement) -> existing)); } - private void appendMetricPriorKnowledge(LLMReq.LLMSchema llmSchema, + private void appendMetricPriorKnowledge( + LLMReq.LLMSchema llmSchema, StringBuilder priorKnowledgeBuilder, SemanticSchema semanticSchema) { - Map fieldNameToDataFormatType = getFieldNameToDataFormatTypeMap(semanticSchema); + Map fieldNameToDataFormatType = + getFieldNameToDataFormatTypeMap(semanticSchema); for (SchemaElement schemaElement : llmSchema.getMetrics()) { String fieldName = schemaElement.getName(); @@ -168,14 +186,15 @@ public class LLMRequestService { private Map getFieldNameToDateFormatMap(SemanticSchema semanticSchema) { return semanticSchema.getDimensions().stream() .filter(dimension -> StringUtils.isNotBlank(dimension.getTimeFormat())) - .collect(Collectors.toMap( - SchemaElement::getName, - value -> Optional.ofNullable(value.getTimeFormat()).orElse(""), - (k1, k2) -> k1) - ); + .collect( + Collectors.toMap( + SchemaElement::getName, + value -> Optional.ofNullable(value.getTimeFormat()).orElse(""), + (k1, k2) -> k1)); } - private void appendDimensionPriorKnowledge(LLMReq.LLMSchema llmSchema, + private void appendDimensionPriorKnowledge( + LLMReq.LLMSchema llmSchema, StringBuilder priorKnowledgeBuilder, SemanticSchema semanticSchema) { Map fieldNameToDateFormat = getFieldNameToDateFormatMap(semanticSchema); @@ -187,7 +206,8 @@ public class LLMRequestService { continue; } if (schemaElement.containsPartitionTime()) { - priorKnowledgeBuilder.append(String.format("%s 是分区时间且格式是%s", fieldName, timeFormat)); + priorKnowledgeBuilder.append( + String.format("%s 是分区时间且格式是%s", fieldName, timeFormat)); } else { priorKnowledgeBuilder.append(String.format("%s 的时间格式是%s", fieldName, timeFormat)); } @@ -195,50 +215,66 @@ public class LLMRequestService { } public List getValues(ChatQueryContext queryCtx, Long dataSetId) { - List matchedElements = queryCtx.getMapInfo().getMatchedElements(dataSetId); + List matchedElements = + queryCtx.getMapInfo().getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(matchedElements)) { return new ArrayList<>(); } - Set valueMatches = matchedElements - .stream() - .filter(elementMatch -> !elementMatch.isInherited()) - .filter(schemaElementMatch -> { - SchemaElementType type = schemaElementMatch.getElement().getType(); - return SchemaElementType.VALUE.equals(type) || SchemaElementType.ID.equals(type); - }) - .map(elementMatch -> { - LLMReq.ElementValue elementValue = new LLMReq.ElementValue(); - elementValue.setFieldName(elementMatch.getElement().getName()); - elementValue.setFieldValue(elementMatch.getWord()); - return elementValue; - }).collect(Collectors.toSet()); + Set valueMatches = + matchedElements.stream() + .filter(elementMatch -> !elementMatch.isInherited()) + .filter( + schemaElementMatch -> { + SchemaElementType type = + schemaElementMatch.getElement().getType(); + return SchemaElementType.VALUE.equals(type) + || SchemaElementType.ID.equals(type); + }) + .map( + elementMatch -> { + LLMReq.ElementValue elementValue = new LLMReq.ElementValue(); + elementValue.setFieldName(elementMatch.getElement().getName()); + elementValue.setFieldValue(elementMatch.getWord()); + return elementValue; + }) + .collect(Collectors.toSet()); return new ArrayList<>(valueMatches); } protected List getMatchedMetrics(ChatQueryContext queryCtx, Long dataSetId) { - List matchedElements = queryCtx.getMapInfo().getMatchedElements(dataSetId); + List matchedElements = + queryCtx.getMapInfo().getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(matchedElements)) { return Collections.emptyList(); } - List schemaElements = matchedElements.stream() - .filter(schemaElementMatch -> { - SchemaElementType elementType = schemaElementMatch.getElement().getType(); - return SchemaElementType.METRIC.equals(elementType); - }) - .map(schemaElementMatch -> { - return schemaElementMatch.getElement(); - }) - .collect(Collectors.toList()); + List schemaElements = + matchedElements.stream() + .filter( + schemaElementMatch -> { + SchemaElementType elementType = + schemaElementMatch.getElement().getType(); + return SchemaElementType.METRIC.equals(elementType); + }) + .map( + schemaElementMatch -> { + return schemaElementMatch.getElement(); + }) + .collect(Collectors.toList()); return schemaElements; } protected List getMatchedDimensions(ChatQueryContext queryCtx, Long dataSetId) { - List matchedElements = queryCtx.getMapInfo().getMatchedElements(dataSetId); - Set dimensionElements = matchedElements.stream() - .filter(element -> SchemaElementType.DIMENSION.equals(element.getElement().getType())) - .map(SchemaElementMatch::getElement) - .collect(Collectors.toSet()); + List matchedElements = + queryCtx.getMapInfo().getMatchedElements(dataSetId); + Set dimensionElements = + matchedElements.stream() + .filter( + element -> + SchemaElementType.DIMENSION.equals( + element.getElement().getType())) + .map(SchemaElementMatch::getElement) + .collect(Collectors.toSet()); SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); if (semanticSchema == null || semanticSchema.getDataSetSchemaMap() == null) { return new ArrayList<>(dimensionElements); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java index 03dc92a6b..e69e73fab 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java @@ -1,20 +1,20 @@ package com.tencent.supersonic.headless.chat.parser.llm; -import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.jsqlparser.SqlValidHelper; +import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Text2SQLExemplar; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.QueryManager; import com.tencent.supersonic.headless.chat.query.llm.LLMSemanticQuery; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlResp; -import com.tencent.supersonic.headless.chat.ChatQueryContext; -import java.util.ArrayList; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.MapUtils; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -23,25 +23,28 @@ import java.util.Objects; @Service public class LLMResponseService { - public SemanticParseInfo addParseInfo(ChatQueryContext queryCtx, ParseResult parseResult, - String s2SQL, Double weight) { + public SemanticParseInfo addParseInfo( + ChatQueryContext queryCtx, ParseResult parseResult, String s2SQL, Double weight) { if (Objects.isNull(weight)) { weight = 0D; } LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(LLMSqlQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); parseInfo.setDataSet(queryCtx.getSemanticSchema().getDataSet(parseResult.getDataSetId())); - parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getDataSetId())); + parseInfo + .getElementMatches() + .addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getDataSetId())); Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, parseResult); properties.put("type", "internal"); - Text2SQLExemplar exemplar = Text2SQLExemplar.builder() - .question(queryCtx.getQueryText()) - .sideInfo(parseResult.getLlmResp().getSideInfo()) - .dbSchema(parseResult.getLlmResp().getSchema()) - .sql(parseResult.getLlmResp().getSqlOutput()) - .build(); + Text2SQLExemplar exemplar = + Text2SQLExemplar.builder() + .question(queryCtx.getQueryText()) + .sideInfo(parseResult.getLlmResp().getSideInfo()) + .dbSchema(parseResult.getLlmResp().getSchema()) + .sql(parseResult.getLlmResp().getSqlOutput()) + .build(); properties.put(Text2SQLExemplar.PROPERTY_KEY, exemplar); parseInfo.setProperties(properties); parseInfo.setScore(queryCtx.getQueryText().length() * (1 + weight)); @@ -60,7 +63,8 @@ public class LLMResponseService { Map result = new HashMap<>(); for (Map.Entry entry : sqlRespMap.entrySet()) { String key = entry.getKey(); - if (result.keySet().stream().anyMatch(existKey -> SqlValidHelper.equals(existKey, key))) { + if (result.keySet().stream() + .anyMatch(existKey -> SqlValidHelper.equals(existKey, key))) { continue; } if (!SqlValidHelper.isValidSQL(key)) { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java index ba44f0305..55eb051af 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java @@ -1,22 +1,22 @@ package com.tencent.supersonic.headless.chat.parser.llm; - import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.parser.SemanticParser; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.MapUtils; + import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.MapUtils; /** - * LLMSqlParser uses large language model to understand query semantics and - * generate S2SQL statements to be executed by the semantic query engine. + * LLMSqlParser uses large language model to understand query semantics and generate S2SQL + * statements to be executed by the semantic query engine. */ @Slf4j public class LLMSqlParser implements SemanticParser { @@ -25,18 +25,18 @@ public class LLMSqlParser implements SemanticParser { public void parse(ChatQueryContext queryCtx) { try { LLMRequestService requestService = ContextUtils.getBean(LLMRequestService.class); - //1.determine whether to skip this parser. + // 1.determine whether to skip this parser. if (requestService.isSkip(queryCtx)) { return; } - //2.get dataSetId from queryCtx and chatCtx. + // 2.get dataSetId from queryCtx and chatCtx. Long dataSetId = requestService.getDataSetId(queryCtx); if (dataSetId == null) { return; } log.info("try generating query statement for dataSetId:{}", dataSetId); - //3.invoke LLM service to do parsing. + // 3.invoke LLM service to do parsing. tryParse(queryCtx, dataSetId); } catch (Exception e) { log.error("failed to parse query:", e); @@ -58,11 +58,16 @@ public class LLMSqlParser implements SemanticParser { try { LLMResp llmResp = requestService.runText2SQL(llmReq); if (Objects.nonNull(llmResp)) { - //deduplicate the S2SQL result list and build parserInfo + // deduplicate the S2SQL result list and build parserInfo sqlRespMap = responseService.getDeduplicationSqlResp(currentRetry, llmResp); if (MapUtils.isNotEmpty(sqlRespMap)) { - parseResult = ParseResult.builder().dataSetId(dataSetId).llmReq(llmReq) - .llmResp(llmResp).linkingValues(llmReq.getLinking()).build(); + parseResult = + ParseResult.builder() + .dataSetId(dataSetId) + .llmReq(llmReq) + .llmResp(llmResp) + .linkingValues(llmReq.getLinking()) + .build(); break; } } @@ -80,5 +85,4 @@ public class LLMSqlParser implements SemanticParser { responseService.addParseInfo(queryCtx, parseResult, sql, sqlWeight); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java index 79145820c..d3a2179b1 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java @@ -20,35 +20,35 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; - @Service @Slf4j public class OnePassSCSqlGenStrategy extends SqlGenStrategy { - private static final String INSTRUCTION = "" - + "\n#Role: You are a data analyst experienced in SQL languages." - + "#Task: You will be provided with a natural language question asked by users," - + "please convert it to a SQL query so that relevant data could be returned " - + "by executing the SQL query against underlying database." - + "\n#Rules:" - + "1.ALWAYS generate column specified in the `Schema`, DO NOT hallucinate." - + "2.ALWAYS specify date filter using `>`,`<`,`>=`,`<=` operator." - + "3.ALWAYS calculate the absolute date range by yourself." - + "4.DO NOT include date filter in the where clause if not explicitly expressed in the `Question`." - + "5.DO NOT miss the AGGREGATE operator of metrics, always add it if needed." - + "6.ONLY respond with the converted SQL statement." - + "\n#Exemplars:\n{{exemplar}}" - + "Question:{{question}},Schema:{{schema}},SideInfo:{{information}},SQL:"; + private static final String INSTRUCTION = + "" + + "\n#Role: You are a data analyst experienced in SQL languages." + + "#Task: You will be provided with a natural language question asked by users," + + "please convert it to a SQL query so that relevant data could be returned " + + "by executing the SQL query against underlying database." + + "\n#Rules:" + + "1.ALWAYS generate column specified in the `Schema`, DO NOT hallucinate." + + "2.ALWAYS specify date filter using `>`,`<`,`>=`,`<=` operator." + + "3.ALWAYS calculate the absolute date range by yourself." + + "4.DO NOT include date filter in the where clause if not explicitly expressed in the `Question`." + + "5.DO NOT miss the AGGREGATE operator of metrics, always add it if needed." + + "6.ONLY respond with the converted SQL statement." + + "\n#Exemplars:\n{{exemplar}}" + + "Question:{{question}},Schema:{{schema}},SideInfo:{{information}},SQL:"; @Override public LLMResp generate(LLMReq llmReq) { LLMResp llmResp = new LLMResp(); llmResp.setQuery(llmReq.getQueryText()); - //1.recall exemplars + // 1.recall exemplars keyPipelineLog.info("OnePassSCSqlGenStrategy llmReq:\n{}", llmReq); List> exemplarsList = promptHelper.getFewShotExemplars(llmReq); - //2.generate sql generation prompt for each self-consistency inference + // 2.generate sql generation prompt for each self-consistency inference Map> prompt2Exemplar = new HashMap<>(); for (List exemplars : exemplarsList) { llmReq.setDynamicExemplars(exemplars); @@ -56,25 +56,36 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy { prompt2Exemplar.put(prompt, exemplars); } - //3.perform multiple self-consistency inferences parallelly + // 3.perform multiple self-consistency inferences parallelly Map output2Prompt = new ConcurrentHashMap<>(); - prompt2Exemplar.keySet().parallelStream().forEach(prompt -> { - keyPipelineLog.info("OnePassSCSqlGenStrategy reqPrompt:\n{}", prompt.toUserMessage()); - ChatLanguageModel chatLanguageModel = getChatLanguageModel(llmReq.getModelConfig()); - Response response = chatLanguageModel.generate(prompt.toUserMessage()); - String sqlOutput = StringUtils.normalizeSpace(response.content().text()); - // replace ``` - String sqlOutputFormat = sqlOutput.replaceAll("(?s)```sql\\s*(.*?)\\s*```", "$1").trim(); - output2Prompt.put(sqlOutputFormat, prompt); - keyPipelineLog.info("OnePassSCSqlGenStrategy modelResp:\n{}", sqlOutputFormat); - } - ); + prompt2Exemplar + .keySet() + .parallelStream() + .forEach( + prompt -> { + keyPipelineLog.info( + "OnePassSCSqlGenStrategy reqPrompt:\n{}", + prompt.toUserMessage()); + ChatLanguageModel chatLanguageModel = + getChatLanguageModel(llmReq.getModelConfig()); + Response response = + chatLanguageModel.generate(prompt.toUserMessage()); + String sqlOutput = + StringUtils.normalizeSpace(response.content().text()); + // replace ``` + String sqlOutputFormat = + sqlOutput.replaceAll("(?s)```sql\\s*(.*?)\\s*```", "$1").trim(); + output2Prompt.put(sqlOutputFormat, prompt); + keyPipelineLog.info( + "OnePassSCSqlGenStrategy modelResp:\n{}", sqlOutputFormat); + }); - //4.format response. - Pair> sqlMapPair = ResponseHelper.selfConsistencyVote( - Lists.newArrayList(output2Prompt.keySet())); + // 4.format response. + Pair> sqlMapPair = + ResponseHelper.selfConsistencyVote(Lists.newArrayList(output2Prompt.keySet())); llmResp.setSqlOutput(sqlMapPair.getLeft()); - List usedExemplars = prompt2Exemplar.get(output2Prompt.get(sqlMapPair.getLeft())); + List usedExemplars = + prompt2Exemplar.get(output2Prompt.get(sqlMapPair.getLeft())); llmResp.setSqlRespMap(ResponseHelper.buildSqlRespMap(usedExemplars, sqlMapPair.getRight())); return llmResp; @@ -83,9 +94,13 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy { private Prompt generatePrompt(LLMReq llmReq, LLMResp llmResp) { StringBuilder exemplars = new StringBuilder(); for (Text2SQLExemplar exemplar : llmReq.getDynamicExemplars()) { - String exemplarStr = String.format("Question:%s,Schema:%s,SideInfo:%s,SQL:%s\n", - exemplar.getQuestion(), exemplar.getDbSchema(), - exemplar.getSideInfo(), exemplar.getSql()); + String exemplarStr = + String.format( + "Question:%s,Schema:%s,SideInfo:%s,SQL:%s\n", + exemplar.getQuestion(), + exemplar.getDbSchema(), + exemplar.getSideInfo(), + exemplar.getSql()); exemplars.append(exemplarStr); } String dataSemantics = promptHelper.buildSchemaStr(llmReq); @@ -110,6 +125,7 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy { @Override public void afterPropertiesSet() { - SqlGenStrategyFactory.addSqlGenerationForFactory(LLMReq.SqlGenType.ONE_PASS_SELF_CONSISTENCY, this); + SqlGenStrategyFactory.addSqlGenerationForFactory( + LLMReq.SqlGenType.ONE_PASS_SELF_CONSISTENCY, this); } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java index 56d91671b..cefe470fe 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java @@ -23,21 +23,23 @@ import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_SE @Slf4j public class PromptHelper { - @Autowired - private ParserConfig parserConfig; + @Autowired private ParserConfig parserConfig; - @Autowired - private ExemplarService exemplarService; + @Autowired private ExemplarService exemplarService; public List> getFewShotExemplars(LLMReq llmReq) { - int exemplarRecallNumber = Integer.valueOf(parserConfig.getParameterValue(PARSER_EXEMPLAR_RECALL_NUMBER)); + int exemplarRecallNumber = + Integer.valueOf(parserConfig.getParameterValue(PARSER_EXEMPLAR_RECALL_NUMBER)); int fewShotNumber = Integer.valueOf(parserConfig.getParameterValue(PARSER_FEW_SHOT_NUMBER)); - int selfConsistencyNumber = Integer.valueOf(parserConfig.getParameterValue(PARSER_SELF_CONSISTENCY_NUMBER)); + int selfConsistencyNumber = + Integer.valueOf(parserConfig.getParameterValue(PARSER_SELF_CONSISTENCY_NUMBER)); List exemplars = Lists.newArrayList(); - llmReq.getDynamicExemplars().stream().forEach(e -> { - exemplars.add(e); - }); + llmReq.getDynamicExemplars().stream() + .forEach( + e -> { + exemplars.add(e); + }); int recallSize = exemplarRecallNumber - llmReq.getDynamicExemplars().size(); if (recallSize > 0) { @@ -76,73 +78,83 @@ public class PromptHelper { String tableStr = llmReq.getSchema().getDataSetName(); List metrics = Lists.newArrayList(); - llmReq.getSchema().getMetrics().stream().forEach( - metric -> { - StringBuilder metricStr = new StringBuilder(); - metricStr.append("<"); - metricStr.append(metric.getName()); - if (!CollectionUtils.isEmpty(metric.getAlias())) { - StringBuilder alias = new StringBuilder(); - metric.getAlias().stream().forEach(a -> alias.append(a + ",")); - metricStr.append(" ALIAS '" + alias + "'"); - } - if (StringUtils.isNotEmpty(metric.getDescription())) { - metricStr.append(" COMMENT '" + metric.getDescription() + "'"); - } - if (StringUtils.isNotEmpty(metric.getDefaultAgg())) { - metricStr.append(" AGGREGATE '" + metric.getDefaultAgg().toUpperCase() + "'"); - } - metricStr.append(">"); - metrics.add(metricStr.toString()); - } - ); + llmReq.getSchema().getMetrics().stream() + .forEach( + metric -> { + StringBuilder metricStr = new StringBuilder(); + metricStr.append("<"); + metricStr.append(metric.getName()); + if (!CollectionUtils.isEmpty(metric.getAlias())) { + StringBuilder alias = new StringBuilder(); + metric.getAlias().stream().forEach(a -> alias.append(a + ",")); + metricStr.append(" ALIAS '" + alias + "'"); + } + if (StringUtils.isNotEmpty(metric.getDescription())) { + metricStr.append(" COMMENT '" + metric.getDescription() + "'"); + } + if (StringUtils.isNotEmpty(metric.getDefaultAgg())) { + metricStr.append( + " AGGREGATE '" + + metric.getDefaultAgg().toUpperCase() + + "'"); + } + metricStr.append(">"); + metrics.add(metricStr.toString()); + }); List dimensions = Lists.newArrayList(); - llmReq.getSchema().getDimensions().stream().forEach( - dimension -> { - StringBuilder dimensionStr = new StringBuilder(); - dimensionStr.append("<"); - dimensionStr.append(dimension.getName()); - if (!CollectionUtils.isEmpty(dimension.getAlias())) { - StringBuilder alias = new StringBuilder(); - dimension.getAlias().stream().forEach(a -> alias.append(a + ",")); - dimensionStr.append(" ALIAS '" + alias + "'"); - } - if (StringUtils.isNotEmpty(dimension.getDescription())) { - dimensionStr.append(" COMMENT '" + dimension.getDescription() + "'"); - } - dimensionStr.append(">"); - dimensions.add(dimensionStr.toString()); - } - ); + llmReq.getSchema().getDimensions().stream() + .forEach( + dimension -> { + StringBuilder dimensionStr = new StringBuilder(); + dimensionStr.append("<"); + dimensionStr.append(dimension.getName()); + if (!CollectionUtils.isEmpty(dimension.getAlias())) { + StringBuilder alias = new StringBuilder(); + dimension.getAlias().stream().forEach(a -> alias.append(a + ",")); + dimensionStr.append(" ALIAS '" + alias + "'"); + } + if (StringUtils.isNotEmpty(dimension.getDescription())) { + dimensionStr.append( + " COMMENT '" + dimension.getDescription() + "'"); + } + dimensionStr.append(">"); + dimensions.add(dimensionStr.toString()); + }); List values = Lists.newArrayList(); - llmReq.getLinking().stream().forEach( - value -> { - StringBuilder valueStr = new StringBuilder(); - String fieldName = value.getFieldName(); - String fieldValue = value.getFieldValue(); - valueStr.append(String.format("<%s='%s'>", fieldName, fieldValue)); - values.add(valueStr.toString()); - } - ); + llmReq.getLinking().stream() + .forEach( + value -> { + StringBuilder valueStr = new StringBuilder(); + String fieldName = value.getFieldName(); + String fieldValue = value.getFieldValue(); + valueStr.append(String.format("<%s='%s'>", fieldName, fieldValue)); + values.add(valueStr.toString()); + }); String template = "Table=[%s], Metrics=[%s], Dimensions=[%s], Values=[%s]"; - return String.format(template, tableStr, String.join(",", metrics), - String.join(",", dimensions), String.join(",", values)); + return String.format( + template, + tableStr, + String.join(",", metrics), + String.join(",", dimensions), + String.join(",", values)); } private String buildTermStr(LLMReq llmReq) { List terms = llmReq.getSchema().getTerms(); List termStr = Lists.newArrayList(); - terms.stream().forEach( - term -> { - StringBuilder termsDesc = new StringBuilder(); - String description = term.getDescription(); - termsDesc.append(String.format("<%s COMMENT '%s'>", term.getName(), description)); - termStr.add(termsDesc.toString()); - } - ); + terms.stream() + .forEach( + term -> { + StringBuilder termsDesc = new StringBuilder(); + String description = term.getDescription(); + termsDesc.append( + String.format( + "<%s COMMENT '%s'>", term.getName(), description)); + termStr.add(termsDesc.toString()); + }); String ret = ""; if (termStr.size() > 0) { ret = String.join(",", termStr); @@ -150,5 +162,4 @@ public class PromptHelper { return ret; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/ResponseHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/ResponseHelper.java index 40db0567b..3c147965f 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/ResponseHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/ResponseHelper.java @@ -54,16 +54,19 @@ public class ResponseHelper { return Pair.of(inputMax, votePercentage); } - public static Map buildSqlRespMap(List sqlExamples, - Map sqlMap) { + public static Map buildSqlRespMap( + List sqlExamples, Map sqlMap) { if (sqlMap == null) { return new HashMap<>(); } return sqlMap.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> LLMSqlResp.builder().sqlWeight(entry.getValue()).fewShots(sqlExamples).build()) - ); + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + LLMSqlResp.builder() + .sqlWeight(entry.getValue()) + .fewShots(sqlExamples) + .build())); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategy.java index f65b55de9..7f335be60 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategy.java @@ -12,16 +12,15 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; /** - * SqlGenStrategy abstracts generation step so that - * different LLM prompting strategies can be implemented. + * SqlGenStrategy abstracts generation step so that different LLM prompting strategies can be + * implemented. */ @Service public abstract class SqlGenStrategy implements InitializingBean { protected static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); - @Autowired - protected PromptHelper promptHelper; + @Autowired protected PromptHelper promptHelper; protected ChatLanguageModel getChatLanguageModel(ChatModelConfig modelConfig) { return ModelProvider.getChatModel(modelConfig); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategyFactory.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategyFactory.java index 80b363d22..e4f5c867e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategyFactory.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategyFactory.java @@ -7,13 +7,15 @@ import java.util.concurrent.ConcurrentHashMap; public class SqlGenStrategyFactory { - private static Map sqlGenStrategyMap = new ConcurrentHashMap<>(); + private static Map sqlGenStrategyMap = + new ConcurrentHashMap<>(); public static SqlGenStrategy get(LLMReq.SqlGenType strategyType) { return sqlGenStrategyMap.get(strategyType); } - public static void addSqlGenerationForFactory(LLMReq.SqlGenType strategy, SqlGenStrategy sqlGenStrategy) { + public static void addSqlGenerationForFactory( + LLMReq.SqlGenType strategy, SqlGenStrategy sqlGenStrategy) { sqlGenStrategyMap.put(strategy, sqlGenStrategy); } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/AggregateTypeParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/AggregateTypeParser.java index d2095e364..e4efda24a 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/AggregateTypeParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/AggregateTypeParser.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.chat.parser.rule; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.headless.chat.ChatQueryContext; -import com.tencent.supersonic.headless.chat.query.SemanticQuery; import com.tencent.supersonic.headless.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.chat.query.SemanticQuery; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -20,24 +20,34 @@ import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT; import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.DISTINCT; /** - * AggregateTypeParser extracts aggregation type specified in the user query - * based on keyword matching. - * Currently, it supports 7 types of aggregation: max, min, sum, avg, topN, - * distinct count, count. + * AggregateTypeParser extracts aggregation type specified in the user query based on keyword + * matching. Currently, it supports 7 types of aggregation: max, min, sum, avg, topN, distinct + * count, count. */ @Slf4j public class AggregateTypeParser implements SemanticParser { - private static final Map REGX_MAP = Stream.of( - new AbstractMap.SimpleEntry<>(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高|最多)")), - new AbstractMap.SimpleEntry<>(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低|最少)")), - new AbstractMap.SimpleEntry<>(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)")), - new AbstractMap.SimpleEntry<>(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)")), - new AbstractMap.SimpleEntry<>(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)")), - new AbstractMap.SimpleEntry<>(DISTINCT, Pattern.compile("(?i)(uv)")), - new AbstractMap.SimpleEntry<>(COUNT, Pattern.compile("(?i)(总数|pv)")), - new AbstractMap.SimpleEntry<>(AggregateTypeEnum.NONE, Pattern.compile("(?i)(明细)")) - ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (k1, k2) -> k2)); + private static final Map REGX_MAP = + Stream.of( + new AbstractMap.SimpleEntry<>( + AggregateTypeEnum.MAX, + Pattern.compile("(?i)(最大值|最大|max|峰值|最高|最多)")), + new AbstractMap.SimpleEntry<>( + AggregateTypeEnum.MIN, + Pattern.compile("(?i)(最小值|最小|min|最低|最少)")), + new AbstractMap.SimpleEntry<>( + AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)")), + new AbstractMap.SimpleEntry<>( + AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)")), + new AbstractMap.SimpleEntry<>( + AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)")), + new AbstractMap.SimpleEntry<>(DISTINCT, Pattern.compile("(?i)(uv)")), + new AbstractMap.SimpleEntry<>(COUNT, Pattern.compile("(?i)(总数|pv)")), + new AbstractMap.SimpleEntry<>( + AggregateTypeEnum.NONE, Pattern.compile("(?i)(明细)"))) + .collect( + Collectors.toMap( + Map.Entry::getKey, Map.Entry::getValue, (k1, k2) -> k2)); @Override public void parse(ChatQueryContext chatQueryContext) { @@ -53,7 +63,9 @@ public class AggregateTypeParser implements SemanticParser { if (StringUtils.isNotEmpty(aggregateConf.detectWord)) { detectWordLength = aggregateConf.detectWord.length(); } - semanticQuery.getParseInfo().setScore(semanticQuery.getParseInfo().getScore() + detectWordLength); + semanticQuery + .getParseInfo() + .setScore(semanticQuery.getParseInfo().getScore() + detectWordLength); } } @@ -66,7 +78,6 @@ public class AggregateTypeParser implements SemanticParser { Map aggregateCount = new HashMap<>(REGX_MAP.size()); Map aggregateWord = new HashMap<>(REGX_MAP.size()); - for (Map.Entry entry : REGX_MAP.entrySet()) { Matcher matcher = entry.getValue().matcher(queryText); int count = 0; @@ -81,8 +92,11 @@ public class AggregateTypeParser implements SemanticParser { } } - AggregateTypeEnum type = aggregateCount.entrySet().stream().max(Map.Entry.comparingByValue()) - .map(entry -> entry.getKey()).orElse(AggregateTypeEnum.NONE); + AggregateTypeEnum type = + aggregateCount.entrySet().stream() + .max(Map.Entry.comparingByValue()) + .map(entry -> entry.getKey()) + .orElse(AggregateTypeEnum.NONE); String detectWord = aggregateWord.get(type); return new AggregateConf(type, detectWord); } @@ -92,5 +106,4 @@ public class AggregateTypeParser implements SemanticParser { public AggregateTypeEnum type; public String detectWord; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/ContextInheritParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/ContextInheritParser.java index 433263dd0..41b703cd4 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/ContextInheritParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/ContextInheritParser.java @@ -3,14 +3,14 @@ package com.tencent.supersonic.headless.chat.parser.rule; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.chat.ChatQueryContext; +import com.tencent.supersonic.headless.chat.parser.SemanticParser; import com.tencent.supersonic.headless.chat.query.QueryManager; import com.tencent.supersonic.headless.chat.query.SemanticQuery; import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.headless.chat.parser.SemanticParser; import com.tencent.supersonic.headless.chat.query.rule.detail.DetailDimensionQuery; +import com.tencent.supersonic.headless.chat.query.rule.metric.MetricIdQuery; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricSemanticQuery; -import com.tencent.supersonic.headless.chat.query.rule.metric.MetricIdQuery; import lombok.extern.slf4j.Slf4j; import java.util.AbstractMap; @@ -24,23 +24,34 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * ContextInheritParser tries to inherit certain schema elements from context - * so that in multi-turn conversations users don't need to mention some keyword - * repeatedly. + * ContextInheritParser tries to inherit certain schema elements from context so that in multi-turn + * conversations users don't need to mention some keyword repeatedly. */ @Slf4j public class ContextInheritParser implements SemanticParser { - private static final Map> MUTUAL_EXCLUSIVE_MAP = Stream.of( - new AbstractMap.SimpleEntry<>(SchemaElementType.METRIC, Arrays.asList(SchemaElementType.METRIC)), - new AbstractMap.SimpleEntry<>( - SchemaElementType.DIMENSION, Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.VALUE)), - new AbstractMap.SimpleEntry<>( - SchemaElementType.VALUE, Arrays.asList(SchemaElementType.VALUE, SchemaElementType.DIMENSION)), - new AbstractMap.SimpleEntry<>(SchemaElementType.ENTITY, Arrays.asList(SchemaElementType.ENTITY)), - new AbstractMap.SimpleEntry<>(SchemaElementType.DATASET, Arrays.asList(SchemaElementType.DATASET)), - new AbstractMap.SimpleEntry<>(SchemaElementType.ID, Arrays.asList(SchemaElementType.ID)) - ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private static final Map> MUTUAL_EXCLUSIVE_MAP = + Stream.of( + new AbstractMap.SimpleEntry<>( + SchemaElementType.METRIC, + Arrays.asList(SchemaElementType.METRIC)), + new AbstractMap.SimpleEntry<>( + SchemaElementType.DIMENSION, + Arrays.asList( + SchemaElementType.DIMENSION, SchemaElementType.VALUE)), + new AbstractMap.SimpleEntry<>( + SchemaElementType.VALUE, + Arrays.asList( + SchemaElementType.VALUE, SchemaElementType.DIMENSION)), + new AbstractMap.SimpleEntry<>( + SchemaElementType.ENTITY, + Arrays.asList(SchemaElementType.ENTITY)), + new AbstractMap.SimpleEntry<>( + SchemaElementType.DATASET, + Arrays.asList(SchemaElementType.DATASET)), + new AbstractMap.SimpleEntry<>( + SchemaElementType.ID, Arrays.asList(SchemaElementType.ID))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @Override public void parse(ChatQueryContext chatQueryContext) { @@ -52,14 +63,17 @@ public class ContextInheritParser implements SemanticParser { return; } - List elementMatches = chatQueryContext.getMapInfo().getMatchedElements(dataSetId); + List elementMatches = + chatQueryContext.getMapInfo().getMatchedElements(dataSetId); List matchesToInherit = new ArrayList<>(); - for (SchemaElementMatch match : chatQueryContext.getContextParseInfo().getElementMatches()) { + for (SchemaElementMatch match : + chatQueryContext.getContextParseInfo().getElementMatches()) { SchemaElementType matchType = match.getElement().getType(); // mutual exclusive element types should not be inherited - RuleSemanticQuery ruleQuery = QueryManager.getRuleQuery( - chatQueryContext.getContextParseInfo().getQueryMode()); + RuleSemanticQuery ruleQuery = + QueryManager.getRuleQuery( + chatQueryContext.getContextParseInfo().getQueryMode()); if (!containsTypes(elementMatches, matchType, ruleQuery)) { match.setInherited(true); matchesToInherit.add(match); @@ -67,17 +81,20 @@ public class ContextInheritParser implements SemanticParser { } elementMatches.addAll(matchesToInherit); - List queries = RuleSemanticQuery.resolve(dataSetId, elementMatches, chatQueryContext); + List queries = + RuleSemanticQuery.resolve(dataSetId, elementMatches, chatQueryContext); for (RuleSemanticQuery query : queries) { query.fillParseInfo(chatQueryContext); - if (existSameQuery(query.getParseInfo().getDataSetId(), query.getQueryMode(), chatQueryContext)) { + if (existSameQuery( + query.getParseInfo().getDataSetId(), query.getQueryMode(), chatQueryContext)) { continue; } chatQueryContext.getCandidateQueries().add(query); } } - private boolean existSameQuery(Long dataSetId, String queryMode, ChatQueryContext chatQueryContext) { + private boolean existSameQuery( + Long dataSetId, String queryMode, ChatQueryContext chatQueryContext) { for (SemanticQuery semanticQuery : chatQueryContext.getCandidateQueries()) { if (semanticQuery.getQueryMode().equals(queryMode) && semanticQuery.getParseInfo().getDataSetId().equals(dataSetId)) { @@ -87,26 +104,34 @@ public class ContextInheritParser implements SemanticParser { return false; } - private boolean containsTypes(List matches, SchemaElementType matchType, + private boolean containsTypes( + List matches, + SchemaElementType matchType, RuleSemanticQuery ruleQuery) { List types = MUTUAL_EXCLUSIVE_MAP.get(matchType); - return matches.stream().anyMatch(m -> { - SchemaElementType type = m.getElement().getType(); - if (Objects.nonNull(ruleQuery) && ruleQuery instanceof MetricSemanticQuery - && !(ruleQuery instanceof MetricIdQuery)) { - return types.contains(type); - } - return type.equals(matchType); - }); + return matches.stream() + .anyMatch( + m -> { + SchemaElementType type = m.getElement().getType(); + if (Objects.nonNull(ruleQuery) + && ruleQuery instanceof MetricSemanticQuery + && !(ruleQuery instanceof MetricIdQuery)) { + return types.contains(type); + } + return type.equals(matchType); + }); } protected boolean shouldInherit(ChatQueryContext chatQueryContext) { // if candidates only have MetricModel mode, count in context - List metricModelQueries = chatQueryContext.getCandidateQueries().stream() - .filter(query -> query instanceof MetricModelQuery - || query instanceof DetailDimensionQuery).collect( - Collectors.toList()); + List metricModelQueries = + chatQueryContext.getCandidateQueries().stream() + .filter( + query -> + query instanceof MetricModelQuery + || query instanceof DetailDimensionQuery) + .collect(Collectors.toList()); return metricModelQueries.size() == chatQueryContext.getCandidateQueries().size(); } @@ -121,5 +146,4 @@ public class ContextInheritParser implements SemanticParser { } return dataSetId; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/RuleSqlParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/RuleSqlParser.java index 3513cecae..cd107d702 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/RuleSqlParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/RuleSqlParser.java @@ -3,24 +3,23 @@ package com.tencent.supersonic.headless.chat.parser.rule; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; -import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.headless.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; + import java.util.Arrays; import java.util.List; /** - * RuleSqlParser resolves a specific SemanticQuery according to co-appearance - * of certain schema element types. + * RuleSqlParser resolves a specific SemanticQuery according to co-appearance of certain schema + * element types. */ @Slf4j public class RuleSqlParser implements SemanticParser { - private static List auxiliaryParsers = Arrays.asList( - new ContextInheritParser(), - new TimeRangeParser(), - new AggregateTypeParser() - ); + private static List auxiliaryParsers = + Arrays.asList( + new ContextInheritParser(), new TimeRangeParser(), new AggregateTypeParser()); @Override public void parse(ChatQueryContext chatQueryContext) { @@ -31,7 +30,8 @@ public class RuleSqlParser implements SemanticParser { // iterate all schemaElementMatches to resolve query mode for (Long dataSetId : mapInfo.getMatchedDataSetInfos()) { List elementMatches = mapInfo.getMatchedElements(dataSetId); - List queries = RuleSemanticQuery.resolve(dataSetId, elementMatches, chatQueryContext); + List queries = + RuleSemanticQuery.resolve(dataSetId, elementMatches, chatQueryContext); for (RuleSemanticQuery query : queries) { query.fillParseInfo(chatQueryContext); chatQueryContext.getCandidateQueries().add(query); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java index 58764405a..9bc996504 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java @@ -23,17 +23,16 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; /** - * TimeRangeParser extracts time range specified in the user query - * based on keyword matching. - * Currently, it supports two kinds of expression: - * 1. Recent unit: 近N天/周/月/年、过去N天/周/月/年 - * 2. Concrete date: 2023年11月15日、20231115 + * TimeRangeParser extracts time range specified in the user query based on keyword matching. + * Currently, it supports two kinds of expression: 1. Recent unit: 近N天/周/月/年、过去N天/周/月/年 2. Concrete + * date: 2023年11月15日、20231115 */ @Slf4j public class TimeRangeParser implements SemanticParser { - private static final Pattern RECENT_PATTERN_CN = Pattern.compile( - ".*(?(近|过去)((?\\d+)|(?[一二三四五六七八九十百千万亿]+))个?(?[天周月年])).*"); + private static final Pattern RECENT_PATTERN_CN = + Pattern.compile( + ".*(?(近|过去)((?\\d+)|(?[一二三四五六七八九十百千万亿]+))个?(?[天周月年])).*"); private static final Pattern DATE_PATTERN_NUMBER = Pattern.compile("(\\d{8})"); private static final DateFormat DATE_FORMAT_NUMBER = new SimpleDateFormat("yyyyMMdd"); private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); @@ -66,11 +65,13 @@ public class TimeRangeParser implements SemanticParser { } else { SemanticParseInfo contextParseInfo = queryContext.getContextParseInfo(); if (QueryManager.containsRuleQuery(contextParseInfo.getQueryMode())) { - RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery(contextParseInfo.getQueryMode()); + RuleSemanticQuery semanticQuery = + QueryManager.createRuleQuery(contextParseInfo.getQueryMode()); if (queryContext.containsPartitionDimensions(contextParseInfo.getDataSetId())) { contextParseInfo.setDateInfo(dateConf); } - contextParseInfo.setScore(contextParseInfo.getScore() + dateConf.getDetectWord().length()); + contextParseInfo.setScore( + contextParseInfo.getScore() + dateConf.getDetectWord().length()); semanticQuery.setParseInfo(contextParseInfo); queryContext.getCandidateQueries().add(semanticQuery); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/BaseSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/BaseSemanticQuery.java index bd2cca913..2093c646c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/BaseSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/BaseSemanticQuery.java @@ -1,4 +1,3 @@ - package com.tencent.supersonic.headless.chat.query; import com.tencent.supersonic.common.pojo.Aggregator; @@ -53,7 +52,8 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { parseInfo.getSqlInfo().setCorrectedS2SQL(querySQLReq.getSql()); } - protected void convertBizNameToName(DataSetSchema dataSetSchema, QueryStructReq queryStructReq) { + protected void convertBizNameToName( + DataSetSchema dataSetSchema, QueryStructReq queryStructReq) { Map bizNameToName = dataSetSchema.getBizNameToName(); bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap()); @@ -76,12 +76,12 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } List dimensionFilters = queryStructReq.getDimensionFilters(); if (CollectionUtils.isNotEmpty(dimensionFilters)) { - dimensionFilters.forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); + dimensionFilters.forEach( + filter -> filter.setName(bizNameToName.get(filter.getBizName()))); } List metricFilters = queryStructReq.getMetricFilters(); if (CollectionUtils.isNotEmpty(dimensionFilters)) { metricFilters.forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/QueryManager.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/QueryManager.java index 436f40df6..752239e46 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/QueryManager.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/QueryManager.java @@ -2,8 +2,9 @@ package com.tencent.supersonic.headless.chat.query; import com.tencent.supersonic.headless.chat.query.llm.LLMSemanticQuery; import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.headless.chat.query.rule.metric.MetricSemanticQuery; import com.tencent.supersonic.headless.chat.query.rule.detail.DetailSemanticQuery; +import com.tencent.supersonic.headless.chat.query.rule.metric.MetricSemanticQuery; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,7 +29,6 @@ public class QueryManager { return createRuleQuery(queryMode); } return createLLMQuery(queryMode); - } public static RuleSemanticQuery createRuleQuery(String queryMode) { @@ -83,5 +83,4 @@ public class QueryManager { public static List getRuleQueries() { return new ArrayList<>(ruleQueryMap.values()); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/SemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/SemanticQuery.java index bd1faa7a5..18fa89615 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/SemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/SemanticQuery.java @@ -6,9 +6,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import org.apache.calcite.sql.parser.SqlParseException; -/** - * A semantic query executes specific type of query based on the results of semantic parsing. - */ +/** A semantic query executes specific type of query based on the results of semantic parsing. */ public interface SemanticQuery { String getQueryMode(); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/LLMSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/LLMSemanticQuery.java index df8b48745..52f8843a0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/LLMSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/LLMSemanticQuery.java @@ -4,5 +4,4 @@ import com.tencent.supersonic.headless.chat.query.BaseSemanticQuery; import lombok.extern.slf4j.Slf4j; @Slf4j -public abstract class LLMSemanticQuery extends BaseSemanticQuery { -} +public abstract class LLMSemanticQuery extends BaseSemanticQuery {} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java index dddb03cd5..ac86244bd 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java @@ -29,7 +29,6 @@ public class LLMReq { public static class ElementValue { private String fieldName; private String fieldValue; - } @Data @@ -44,10 +43,16 @@ public class LLMReq { public List getFieldNameList() { List fieldNameList = new ArrayList<>(); if (CollectionUtils.isNotEmpty(metrics)) { - fieldNameList.addAll(metrics.stream().map(metric -> metric.getName()).collect(Collectors.toList())); + fieldNameList.addAll( + metrics.stream() + .map(metric -> metric.getName()) + .collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(dimensions)) { - fieldNameList.addAll(dimensions.stream().map(metric -> metric.getName()).collect(Collectors.toList())); + fieldNameList.addAll( + dimensions.stream() + .map(metric -> metric.getName()) + .collect(Collectors.toList())); } return fieldNameList; } @@ -72,6 +77,5 @@ public class LLMReq { public String getName() { return name; } - } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMResp.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMResp.java index 54b11ccd6..8d23c48bc 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMResp.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMResp.java @@ -21,5 +21,4 @@ public class LLMResp { private List fields; private Map sqlRespMap; - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlResp.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlResp.java index f0ab841e2..4a8da3fef 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlResp.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlResp.java @@ -17,5 +17,4 @@ public class LLMSqlResp { private double sqlWeight; private List fewShots; - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatchOption.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatchOption.java index e87d02d46..9e5f9dd6e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatchOption.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatchOption.java @@ -9,8 +9,10 @@ public class QueryMatchOption { private RequireNumberType requireNumberType; private Integer requireNumber; - public static QueryMatchOption build(OptionType schemaElementOption, - RequireNumberType requireNumberType, Integer requireNumber) { + public static QueryMatchOption build( + OptionType schemaElementOption, + RequireNumberType requireNumberType, + Integer requireNumber) { QueryMatchOption queryMatchOption = new QueryMatchOption(); queryMatchOption.requireNumber = requireNumber; queryMatchOption.requireNumberType = requireNumberType; @@ -35,11 +37,14 @@ public class QueryMatchOption { } public enum RequireNumberType { - AT_MOST, AT_LEAST, EQUAL + AT_MOST, + AT_LEAST, + EQUAL } public enum OptionType { - REQUIRED, OPTIONAL, UNUSED + REQUIRED, + OPTIONAL, + UNUSED } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatcher.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatcher.java index 066f42ec3..bd498dc7c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatcher.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/QueryMatcher.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.query.rule; - import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; @@ -21,8 +20,8 @@ public class QueryMatcher { private HashMap elementOptionMap = new HashMap<>(); private boolean supportCompare; private boolean supportOrderBy; - private List orderByTypes = Arrays.asList(AggregateTypeEnum.MAX, AggregateTypeEnum.MIN, - AggregateTypeEnum.TOPN); + private List orderByTypes = + Arrays.asList(AggregateTypeEnum.MAX, AggregateTypeEnum.MIN, AggregateTypeEnum.TOPN); public QueryMatcher() { for (SchemaElementType type : SchemaElementType.values()) { @@ -34,9 +33,13 @@ public class QueryMatcher { } } - public QueryMatcher addOption(SchemaElementType type, QueryMatchOption.OptionType option, - QueryMatchOption.RequireNumberType requireNumberType, Integer requireNumber) { - elementOptionMap.put(type, QueryMatchOption.build(option, requireNumberType, requireNumber)); + public QueryMatcher addOption( + SchemaElementType type, + QueryMatchOption.OptionType option, + QueryMatchOption.RequireNumberType requireNumberType, + Integer requireNumber) { + elementOptionMap.put( + type, QueryMatchOption.build(option, requireNumberType, requireNumber)); return this; } @@ -44,8 +47,7 @@ public class QueryMatcher { * Match schema element with current query according to the options. * * @param candidateElementMatches - * @return a list of all matched schema elements, - * empty list if no matches can be found + * @return a list of all matched schema elements, empty list if no matches can be found */ public List match(List candidateElementMatches) { List elementMatches = new ArrayList<>(); @@ -53,7 +55,8 @@ public class QueryMatcher { for (SchemaElementMatch schemaElementMatch : candidateElementMatches) { SchemaElementType schemaElementType = schemaElementMatch.getElement().getType(); if (schemaElementTypeCount.containsKey(schemaElementType)) { - schemaElementTypeCount.put(schemaElementType, schemaElementTypeCount.get(schemaElementType) + 1); + schemaElementTypeCount.put( + schemaElementType, schemaElementTypeCount.get(schemaElementType) + 1); } else { schemaElementTypeCount.put(schemaElementType, 1); } @@ -70,9 +73,12 @@ public class QueryMatcher { // add element match if its element type is not declared as unused for (SchemaElementMatch elementMatch : candidateElementMatches) { - QueryMatchOption elementOption = elementOptionMap.get(elementMatch.getElement().getType()); - if (Objects.nonNull(elementOption) && !elementOption.getSchemaElementOption() - .equals(QueryMatchOption.OptionType.UNUSED)) { + QueryMatchOption elementOption = + elementOptionMap.get(elementMatch.getElement().getType()); + if (Objects.nonNull(elementOption) + && !elementOption + .getSchemaElementOption() + .equals(QueryMatchOption.OptionType.UNUSED)) { elementMatches.add(elementMatch); } } @@ -80,7 +86,8 @@ public class QueryMatcher { return elementMatches; } - private int getCount(HashMap schemaElementTypeCount, + private int getCount( + HashMap schemaElementTypeCount, SchemaElementType schemaElementType) { if (schemaElementTypeCount.containsKey(schemaElementType)) { return schemaElementTypeCount.get(schemaElementType); @@ -90,14 +97,19 @@ public class QueryMatcher { private boolean isMatch(QueryMatchOption queryMatchOption, int count) { // check if required but empty - if (queryMatchOption.getSchemaElementOption().equals(QueryMatchOption.OptionType.REQUIRED) && count <= 0) { + if (queryMatchOption.getSchemaElementOption().equals(QueryMatchOption.OptionType.REQUIRED) + && count <= 0) { return false; } - if (queryMatchOption.getRequireNumberType().equals(QueryMatchOption.RequireNumberType.AT_LEAST) + if (queryMatchOption + .getRequireNumberType() + .equals(QueryMatchOption.RequireNumberType.AT_LEAST) && count < queryMatchOption.getRequireNumber()) { return false; } - if (queryMatchOption.getRequireNumberType().equals(QueryMatchOption.RequireNumberType.AT_MOST) + if (queryMatchOption + .getRequireNumberType() + .equals(QueryMatchOption.RequireNumberType.AT_MOST) && count > queryMatchOption.getRequireNumber()) { return false; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java index 587fcd1bb..d7bd189d1 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java @@ -1,4 +1,3 @@ - package com.tencent.supersonic.headless.chat.query.rule; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -41,8 +40,8 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { QueryManager.register(this); } - public List match(List candidateElementMatches, - ChatQueryContext queryCtx) { + public List match( + List candidateElementMatches, ChatQueryContext queryCtx) { return queryMatcher.match(candidateElementMatches); } @@ -68,17 +67,19 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { return chatQueryContext.containsPartitionDimensions(dataSetId); } - private void fillDateConfByInherited(SemanticParseInfo queryParseInfo, ChatQueryContext chatQueryContext) { + private void fillDateConfByInherited( + SemanticParseInfo queryParseInfo, ChatQueryContext chatQueryContext) { SemanticParseInfo contextParseInfo = chatQueryContext.getContextParseInfo(); - if (queryParseInfo.getDateInfo() != null || contextParseInfo.getDateInfo() == null + if (queryParseInfo.getDateInfo() != null + || contextParseInfo.getDateInfo() == null || needFillDateConf(chatQueryContext)) { return; } if ((QueryManager.isDetailQuery(queryParseInfo.getQueryMode()) - && QueryManager.isDetailQuery(contextParseInfo.getQueryMode())) + && QueryManager.isDetailQuery(contextParseInfo.getQueryMode())) || (QueryManager.isMetricQuery(queryParseInfo.getQueryMode()) - && QueryManager.isMetricQuery(contextParseInfo.getQueryMode()))) { + && QueryManager.isMetricQuery(contextParseInfo.getQueryMode()))) { // inherit date info from context queryParseInfo.setDateInfo(contextParseInfo.getDateInfo()); queryParseInfo.getDateInfo().setInherited(true); @@ -105,8 +106,11 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { } private void fillSchemaElement(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) { - Set dataSetIds = parseInfo.getElementMatches().stream().map(SchemaElementMatch::getElement) - .map(SchemaElement::getDataSetId).collect(Collectors.toSet()); + Set dataSetIds = + parseInfo.getElementMatches().stream() + .map(SchemaElementMatch::getElement) + .map(SchemaElement::getDataSetId) + .collect(Collectors.toSet()); Long dataSetId = dataSetIds.iterator().next(); parseInfo.setDataSet(semanticSchema.getDataSet(dataSetId)); Map> dim2Values = new HashMap<>(); @@ -117,22 +121,26 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { element.setOrder(1 - schemaMatch.getSimilarity()); switch (element.getType()) { case ID: - SchemaElement entityElement = semanticSchema.getElement(SchemaElementType.ENTITY, element.getId()); + SchemaElement entityElement = + semanticSchema.getElement(SchemaElementType.ENTITY, element.getId()); if (entityElement != null) { if (id2Values.containsKey(element.getId())) { id2Values.get(element.getId()).add(schemaMatch); } else { - id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch))); + id2Values.put( + element.getId(), new ArrayList<>(Arrays.asList(schemaMatch))); } } break; case VALUE: - SchemaElement dimElement = semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId()); + SchemaElement dimElement = + semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId()); if (dimElement != null) { if (dim2Values.containsKey(element.getId())) { dim2Values.get(element.getId()).add(schemaMatch); } else { - dim2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch))); + dim2Values.put( + element.getId(), new ArrayList<>(Arrays.asList(schemaMatch))); } } break; @@ -152,8 +160,11 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { addToFilters(dim2Values, parseInfo, semanticSchema, SchemaElementType.DIMENSION); } - private void addToFilters(Map> id2Values, SemanticParseInfo parseInfo, - SemanticSchema semanticSchema, SchemaElementType entity) { + private void addToFilters( + Map> id2Values, + SemanticParseInfo parseInfo, + SemanticSchema semanticSchema, + SchemaElementType entity) { if (id2Values == null || id2Values.isEmpty()) { return; } @@ -170,7 +181,8 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { dimensionFilter.setName(dimension.getName()); dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); dimensionFilter.setElementID(schemaMatch.getElement().getId()); - parseInfo.setEntity(semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey())); + parseInfo.setEntity( + semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey())); parseInfo.getDimensionFilters().add(dimensionFilter); } else { QueryFilter dimensionFilter = new QueryFilter(); @@ -193,7 +205,8 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public SemanticQueryReq multiStructExecute() { String queryMode = parseInfo.getQueryMode(); - if (parseInfo.getDataSetId() != null || StringUtils.isEmpty(queryMode) + if (parseInfo.getDataSetId() != null + || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); @@ -208,14 +221,18 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { this.parseInfo = parseInfo; } - public static List resolve(Long dataSetId, List candidateElementMatches, - ChatQueryContext chatQueryContext) { + public static List resolve( + Long dataSetId, + List candidateElementMatches, + ChatQueryContext chatQueryContext) { List matchedQueries = new ArrayList<>(); for (RuleSemanticQuery semanticQuery : QueryManager.getRuleQueries()) { - List matches = semanticQuery.match(candidateElementMatches, chatQueryContext); + List matches = + semanticQuery.match(candidateElementMatches, chatQueryContext); if (matches.size() > 0) { - RuleSemanticQuery query = QueryManager.createRuleQuery(semanticQuery.getQueryMode()); + RuleSemanticQuery query = + QueryManager.createRuleQuery(semanticQuery.getQueryMode()); query.getParseInfo().getElementMatches().addAll(matches); matchedQueries.add(query); } @@ -230,5 +247,4 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { protected QueryMultiStructReq convertQueryMultiStruct() { return QueryReqBuilder.buildMultiStructReq(parseInfo); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailDimensionQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailDimensionQuery.java index afb1065e8..192004743 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailDimensionQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailDimensionQuery.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.query.rule.detail; - import org.springframework.stereotype.Component; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; @@ -26,5 +25,4 @@ public class DetailDimensionQuery extends DetailSemanticQuery { public String getQueryMode() { return QUERY_MODE; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java index f8837996b..83150295a 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java @@ -24,5 +24,4 @@ public class DetailFilterQuery extends DetailListQuery { public String getQueryMode() { return QUERY_MODE; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java index 2f88cf5da..b96d5971a 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java @@ -20,5 +20,4 @@ public class DetailIdQuery extends DetailListQuery { public String getQueryMode() { return QUERY_MODE; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java index 6a6673c37..348404143 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java @@ -23,32 +23,51 @@ public abstract class DetailListQuery extends DetailSemanticQuery { this.addEntityDetailAndOrderByMetric(chatQueryContext, parseInfo); } - private void addEntityDetailAndOrderByMetric(ChatQueryContext chatQueryContext, SemanticParseInfo parseInfo) { + private void addEntityDetailAndOrderByMetric( + ChatQueryContext chatQueryContext, SemanticParseInfo parseInfo) { Long dataSetId = parseInfo.getDataSetId(); if (Objects.isNull(dataSetId) || dataSetId <= 0L) { return; } - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); if (dataSetSchema != null && Objects.nonNull(dataSetSchema.getEntity())) { Set dimensions = new LinkedHashSet<>(); Set metrics = new LinkedHashSet<>(); Set orders = new LinkedHashSet<>(); TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig(); - if (tagTypeDefaultConfig != null && tagTypeDefaultConfig.getDefaultDisplayInfo() != null) { - if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { - metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds() - .stream().map(id -> { - SchemaElement metric = dataSetSchema.getElement(SchemaElementType.METRIC, id); - if (metric != null) { - orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)); - } - return metric; - }).filter(Objects::nonNull).collect(Collectors.toSet()); + if (tagTypeDefaultConfig != null + && tagTypeDefaultConfig.getDefaultDisplayInfo() != null) { + if (CollectionUtils.isNotEmpty( + tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { + metrics = + tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds().stream() + .map( + id -> { + SchemaElement metric = + dataSetSchema.getElement( + SchemaElementType.METRIC, id); + if (metric != null) { + orders.add( + new Order( + metric.getBizName(), + Constants.DESC_UPPER)); + } + return metric; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); } - if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { - dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> dataSetSchema.getElement(SchemaElementType.DIMENSION, id)) - .filter(Objects::nonNull).collect(Collectors.toSet()); + if (CollectionUtils.isNotEmpty( + tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { + dimensions = + tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() + .map( + id -> + dataSetSchema.getElement( + SchemaElementType.DIMENSION, id)) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); } } parseInfo.setDimensions(dimensions); @@ -56,5 +75,4 @@ public abstract class DetailListQuery extends DetailSemanticQuery { parseInfo.setOrders(orders); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java index 43ce9d9f7..84fec0787 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java @@ -25,8 +25,8 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { } @Override - public List match(List candidateElementMatches, - ChatQueryContext queryCtx) { + public List match( + List candidateElementMatches, ChatQueryContext queryCtx) { return super.match(candidateElementMatches, queryCtx); } @@ -39,7 +39,8 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { if (!needFillDateConf(chatQueryContext)) { return; } - Map dataSetSchemaMap = chatQueryContext.getSemanticSchema().getDataSetSchemaMap(); + Map dataSetSchemaMap = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap(); DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId()); TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); @@ -63,5 +64,4 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { parseInfo.setDateInfo(dateInfo); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricFilterQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricFilterQuery.java index ec9ed4fce..d8a4e6d83 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricFilterQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricFilterQuery.java @@ -19,7 +19,6 @@ import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - @Slf4j @Component public class MetricFilterQuery extends MetricSemanticQuery { @@ -46,8 +45,7 @@ public class MetricFilterQuery extends MetricSemanticQuery { protected boolean isMultiStructQuery() { Set filterBizName = new HashSet<>(); - parseInfo.getDimensionFilters().forEach(filter -> - filterBizName.add(filter.getBizName())); + parseInfo.getDimensionFilters().forEach(filter -> filterBizName.add(filter.getBizName())); return FilterType.UNION.equals(parseInfo.getFilterType()) && filterBizName.size() > 1; } @@ -73,17 +71,21 @@ public class MetricFilterQuery extends MetricSemanticQuery { log.debug("addDimension before [{}]", queryStructReq.getGroups()); List filters = new ArrayList<>(queryStructReq.getDimensionFilters()); if (onlyOperateInFilter) { - filters = filters.stream().filter(filter - -> filter.getOperator().equals(FilterOperatorEnum.IN)).collect(Collectors.toList()); + filters = + filters.stream() + .filter( + filter -> + filter.getOperator().equals(FilterOperatorEnum.IN)) + .collect(Collectors.toList()); } - filters.forEach(d -> { - if (!dimensions.contains(d.getBizName())) { - dimensions.add(d.getBizName()); - } - }); + filters.forEach( + d -> { + if (!dimensions.contains(d.getBizName())) { + dimensions.add(d.getBizName()); + } + }); queryStructReq.setGroups(dimensions); log.debug("addDimension after [{}]", queryStructReq.getGroups()); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricGroupByQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricGroupByQuery.java index b920bbea0..d2f539125 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricGroupByQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricGroupByQuery.java @@ -23,5 +23,4 @@ public class MetricGroupByQuery extends MetricSemanticQuery { public String getQueryMode() { return QUERY_MODE; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java index 9aeae3153..45cce5e81 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java @@ -28,8 +28,7 @@ public class MetricIdQuery extends MetricSemanticQuery { public MetricIdQuery() { super(); - queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1) - .addOption(ENTITY, REQUIRED, AT_LEAST, 1); + queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1).addOption(ENTITY, REQUIRED, AT_LEAST, 1); } @Override @@ -75,17 +74,21 @@ public class MetricIdQuery extends MetricSemanticQuery { log.info("addDimension before [{}]", queryStructReq.getGroups()); List filters = new ArrayList<>(queryStructReq.getDimensionFilters()); if (onlyOperateInFilter) { - filters = filters.stream().filter(filter - -> filter.getOperator().equals(FilterOperatorEnum.IN)).collect(Collectors.toList()); + filters = + filters.stream() + .filter( + filter -> + filter.getOperator().equals(FilterOperatorEnum.IN)) + .collect(Collectors.toList()); } - filters.forEach(d -> { - if (!dimensions.contains(d.getBizName())) { - dimensions.add(d.getBizName()); - } - }); + filters.forEach( + d -> { + if (!dimensions.contains(d.getBizName())) { + dimensions.add(d.getBizName()); + } + }); queryStructReq.setGroups(dimensions); log.info("addDimension after [{}]", queryStructReq.getGroups()); } } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricModelQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricModelQuery.java index ce1958e1f..935efec29 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricModelQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricModelQuery.java @@ -20,5 +20,4 @@ public class MetricModelQuery extends MetricSemanticQuery { public String getQueryMode() { return QUERY_MODE; } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java index 8e15355bf..0d82bd5a3 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.query.rule.metric; - import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; @@ -29,8 +28,8 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { } @Override - public List match(List candidateElementMatches, - ChatQueryContext queryCtx) { + public List match( + List candidateElementMatches, ChatQueryContext queryCtx) { return super.match(candidateElementMatches, queryCtx); } @@ -46,11 +45,15 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { return; } DataSetSchema dataSetSchema = - chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + chatQueryContext + .getSemanticSchema() + .getDataSetSchemaMap() + .get(parseInfo.getDataSetId()); TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig(); DateConf dateInfo = new DateConf(); - //加上时间!=-1 判断 - if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit()) + // 加上时间!=-1 判断 + if (Objects.nonNull(timeDefaultConfig) + && Objects.nonNull(timeDefaultConfig.getUnit()) && timeDefaultConfig.getUnit() != -1) { int unit = timeDefaultConfig.getUnit(); String startDate = LocalDate.now().plusDays(-unit).toString(); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java index e4306bd5a..f2d92b773 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java @@ -34,8 +34,8 @@ public class MetricTopNQuery extends MetricSemanticQuery { } @Override - public List match(List candidateElementMatches, - ChatQueryContext queryCtx) { + public List match( + List candidateElementMatches, ChatQueryContext queryCtx) { Matcher matcher = INTENT_PATTERN.matcher(queryCtx.getQueryText()); if (matcher.matches()) { return super.match(candidateElementMatches, queryCtx); @@ -59,5 +59,4 @@ public class MetricTopNQuery extends MetricSemanticQuery { SchemaElement metric = parseInfo.getMetrics().iterator().next(); parseInfo.getOrders().add(new Order(metric.getBizName(), Constants.DESC_UPPER)); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/ComponentFactory.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/ComponentFactory.java index 1004e5d94..fe5b32790 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/ComponentFactory.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/ComponentFactory.java @@ -8,9 +8,7 @@ import org.springframework.core.io.support.SpringFactoriesLoader; import java.util.List; import java.util.Objects; -/** - * HeadlessConverter QueryOptimizer QueryExecutor object factory - */ +/** HeadlessConverter QueryOptimizer QueryExecutor object factory */ @Slf4j public class ComponentFactory { @@ -28,14 +26,15 @@ public class ComponentFactory { } private static List init(Class factoryType, List list) { - list.addAll(SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader())); + list.addAll( + SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader())); return list; } private static T init(Class factoryType) { - return SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader()).get(0); + return SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader()) + .get(0); } - } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParser.java index 4aab2bf67..b5c1baf43 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParser.java @@ -13,9 +13,10 @@ public class QueryFilterParser { public static String parse(QueryFilters queryFilters) { try { - List conditions = queryFilters.getFilters().stream() - .map(QueryFilterParser::parseFilter) - .collect(Collectors.toList()); + List conditions = + queryFilters.getFilters().stream() + .map(QueryFilterParser::parseFilter) + .collect(Collectors.toList()); return String.join(" AND ", conditions); } catch (Exception e) { log.error("", e); @@ -35,9 +36,14 @@ public class QueryFilterParser { case BETWEEN: if (value instanceof List && ((List) value).size() == 2) { List values = (List) value; - return column + " BETWEEN " + formatValue(values.get(0)) + " AND " + formatValue(values.get(1)); + return column + + " BETWEEN " + + formatValue(values.get(0)) + + " AND " + + formatValue(values.get(1)); } - throw new IllegalArgumentException("BETWEEN operator requires a list of two values"); + throw new IllegalArgumentException( + "BETWEEN operator requires a list of two values"); case IS_NULL: case IS_NOT_NULL: return column + " " + operator.getValue(); @@ -52,9 +58,8 @@ public class QueryFilterParser { private static String parseList(Object value) { if (value instanceof List) { - return ((List) value).stream() - .map(QueryFilterParser::formatValue) - .collect(Collectors.joining(", ")); + return ((List) value) + .stream().map(QueryFilterParser::formatValue).collect(Collectors.joining(", ")); } throw new IllegalArgumentException("IN and NOT IN operators require a list of values"); } @@ -69,4 +74,4 @@ public class QueryFilterParser { } throw new IllegalArgumentException("Unsupported value type: " + value.getClass()); } -} \ No newline at end of file +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java index 33c724b87..f5d4c15c0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java @@ -47,25 +47,37 @@ public class QueryReqBuilder { List dimensionFilters = getFilters(parseInfo.getDimensionFilters()); queryStructReq.setDimensionFilters(dimensionFilters); - List metricFilters = parseInfo.getMetricFilters().stream() - .map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue())) - .collect(Collectors.toList()); + List metricFilters = + parseInfo.getMetricFilters().stream() + .map( + chatFilter -> + new Filter( + chatFilter.getBizName(), + chatFilter.getOperator(), + chatFilter.getValue())) + .collect(Collectors.toList()); queryStructReq.setMetricFilters(metricFilters); addDateDimension(parseInfo); if (isDateFieldAlreadyPresent(parseInfo, getDateField(parseInfo.getDateInfo()))) { - parseInfo.getDimensions().removeIf(schemaElement -> schemaElement.containsPartitionTime()); + parseInfo + .getDimensions() + .removeIf(schemaElement -> schemaElement.containsPartitionTime()); } - queryStructReq.setGroups(parseInfo.getDimensions().stream().map(SchemaElement::getBizName) - .collect(Collectors.toList())); + queryStructReq.setGroups( + parseInfo.getDimensions().stream() + .map(SchemaElement::getBizName) + .collect(Collectors.toList())); queryStructReq.setLimit(parseInfo.getLimit()); // only one metric is queried at once Set metrics = parseInfo.getMetrics(); if (!CollectionUtils.isEmpty(metrics)) { SchemaElement metricElement = parseInfo.getMetrics().iterator().next(); - Set order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), metricElement); - queryStructReq.setAggregators(getAggregatorByMetric(parseInfo.getAggType(), metricElement)); + Set order = + getOrder(parseInfo.getOrders(), parseInfo.getAggType(), metricElement); + queryStructReq.setAggregators( + getAggregatorByMetric(parseInfo.getAggType(), metricElement)); queryStructReq.setOrders(new ArrayList<>(order)); } @@ -75,15 +87,22 @@ public class QueryReqBuilder { } private static List getFilters(Set queryFilters) { - List dimensionFilters = queryFilters.stream() - .filter(chatFilter -> StringUtils.isNotEmpty(chatFilter.getBizName())) - .map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue())) - .collect(Collectors.toList()); + List dimensionFilters = + queryFilters.stream() + .filter(chatFilter -> StringUtils.isNotEmpty(chatFilter.getBizName())) + .map( + chatFilter -> + new Filter( + chatFilter.getBizName(), + chatFilter.getOperator(), + chatFilter.getValue())) + .collect(Collectors.toList()); return dimensionFilters; } private static void deletionDuplicated(QueryStructReq queryStructReq) { - if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) { + if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) + && queryStructReq.getGroups().size() > 1) { Set groups = new HashSet<>(); groups.addAll(queryStructReq.getGroups()); queryStructReq.getGroups().clear(); @@ -167,17 +186,21 @@ public class QueryReqBuilder { return querySQLReq; } - private static List getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { + private static List getAggregatorByMetric( + AggregateTypeEnum aggregateType, SchemaElement metric) { if (metric == null) { return Collections.emptyList(); } String agg = determineAggregator(aggregateType, metric); - return Collections.singletonList(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg))); + return Collections.singletonList( + new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg))); } - private static String determineAggregator(AggregateTypeEnum aggregateType, SchemaElement metric) { - if (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE) + private static String determineAggregator( + AggregateTypeEnum aggregateType, SchemaElement metric) { + if (aggregateType == null + || aggregateType.equals(AggregateTypeEnum.NONE) || AggOperatorEnum.COUNT_DISTINCT.name().equalsIgnoreCase(metric.getDefaultAgg())) { return StringUtils.defaultIfBlank(metric.getDefaultAgg(), ""); } @@ -209,27 +232,32 @@ public class QueryReqBuilder { private static boolean shouldSkipAddingDateDimension(SemanticParseInfo parseInfo) { return parseInfo.getAggType() != null && (parseInfo.getAggType().equals(AggregateTypeEnum.MAX) - || parseInfo.getAggType().equals(AggregateTypeEnum.MIN)) + || parseInfo.getAggType().equals(AggregateTypeEnum.MIN)) && !CollectionUtils.isEmpty(parseInfo.getDimensions()); } - private static boolean isDateFieldAlreadyPresent(SemanticParseInfo parseInfo, String dateField) { + private static boolean isDateFieldAlreadyPresent( + SemanticParseInfo parseInfo, String dateField) { return parseInfo.getDimensions().stream() .anyMatch(dimension -> dimension.getBizName().equalsIgnoreCase(dateField)); } private static void addDimension(SemanticParseInfo parseInfo, SchemaElement dimension) { - List timeDimensions = Arrays.asList(TimeDimensionEnum.DAY.getName(), - TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.MONTH.getName()); - Set dimensions = parseInfo.getDimensions().stream() - .filter(d -> !timeDimensions.contains(d.getBizName().toLowerCase())) - .collect(Collectors.toSet()); + List timeDimensions = + Arrays.asList( + TimeDimensionEnum.DAY.getName(), + TimeDimensionEnum.WEEK.getName(), + TimeDimensionEnum.MONTH.getName()); + Set dimensions = + parseInfo.getDimensions().stream() + .filter(d -> !timeDimensions.contains(d.getBizName().toLowerCase())) + .collect(Collectors.toSet()); dimensions.add(dimension); parseInfo.setDimensions(dimensions); } - public static Set getOrder(Set existingOrders, - AggregateTypeEnum aggregator, SchemaElement metric) { + public static Set getOrder( + Set existingOrders, AggregateTypeEnum aggregator, SchemaElement metric) { if (existingOrders != null && !existingOrders.isEmpty()) { return existingOrders; } @@ -265,8 +293,8 @@ public class QueryReqBuilder { return dateField; } - public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric, - AggOperatorEnum aggOperatorEnum) { + public static QueryStructReq buildStructRatioReq( + SemanticParseInfo parseInfo, SchemaElement metric, AggOperatorEnum aggOperatorEnum) { QueryStructReq queryStructReq = buildStructReq(parseInfo); queryStructReq.setQueryType(QueryType.METRIC); queryStructReq.setOrders(new ArrayList<>()); @@ -276,5 +304,4 @@ public class QueryReqBuilder { queryStructReq.setAggregators(aggregators); return queryStructReq; } - } diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/AggCorrectorTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/AggCorrectorTest.java index f5ccbd042..311585b4e 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/AggCorrectorTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/AggCorrectorTest.java @@ -27,16 +27,18 @@ class AggCorrectorTest { dataSet.setDataSetId(dataSetId); semanticParseInfo.setDataSet(dataSet); SqlInfo sqlInfo = new SqlInfo(); - String sql = "SELECT 用户, 访问次数 FROM 超音数数据集 WHERE 部门 = 'sales' AND" - + " datediff('day', 数据日期, '2024-06-04') <= 7" - + " GROUP BY 用户 ORDER BY SUM(访问次数) DESC LIMIT 1"; + String sql = + "SELECT 用户, 访问次数 FROM 超音数数据集 WHERE 部门 = 'sales' AND" + + " datediff('day', 数据日期, '2024-06-04') <= 7" + + " GROUP BY 用户 ORDER BY SUM(访问次数) DESC LIMIT 1"; sqlInfo.setParsedS2SQL(sql); sqlInfo.setCorrectedS2SQL(sql); semanticParseInfo.setSqlInfo(sqlInfo); corrector.correct(chatQueryContext, semanticParseInfo); - Assert.assertEquals("SELECT 用户, SUM(访问次数) FROM 超音数数据集 WHERE 部门 = 'sales'" - + " AND datediff('day', 数据日期, '2024-06-04') <= 7 GROUP BY 用户" - + " ORDER BY SUM(访问次数) DESC LIMIT 1", + Assert.assertEquals( + "SELECT 用户, SUM(访问次数) FROM 超音数数据集 WHERE 部门 = 'sales'" + + " AND datediff('day', 数据日期, '2024-06-04') <= 7 GROUP BY 用户" + + " ORDER BY SUM(访问次数) DESC LIMIT 1", semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); } @@ -70,5 +72,4 @@ class AggCorrectorTest { chatQueryContext.setSemanticSchema(semanticSchema); return chatQueryContext; } - } diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrectorTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrectorTest.java index 541e7957a..f3d801ba5 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrectorTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrectorTest.java @@ -9,8 +9,8 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SqlInfo; -import com.tencent.supersonic.headless.chat.parser.llm.ParseResult; import com.tencent.supersonic.headless.chat.ChatQueryContext; +import com.tencent.supersonic.headless.chat.parser.llm.ParseResult; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import org.junit.Assert; import org.junit.jupiter.api.Test; @@ -22,32 +22,33 @@ import java.util.Set; class SchemaCorrectorTest { - private String json = "{\n" - + " \"dataSetId\": 1,\n" - + " \"llmReq\": {\n" - + " \"queryText\": \"xxx2024年播放量最高的十首歌\",\n" - + " \"schema\": {\n" - + " \"dataSetName\": \"歌曲\",\n" - + " \"fieldNameList\": [\n" - + " \"商务组\",\n" - + " \"歌曲名\",\n" - + " \"播放量\",\n" - + " \"播放份额\",\n" - + " \"数据日期\"\n" - + " ]\n" - + " },\n" - + " \"linking\": [\n" - + "\n" - + " ],\n" - + " \"currentDate\": \"2024-02-24\",\n" - + " \"priorExts\": \"播放份额是小数; \",\n" - + " \"sqlGenType\": \"1_pass_self_consistency\"\n" - + " },\n" - + " \"request\": null,\n" - + " \"linkingValues\": [\n" - + "\n" - + " ]\n" - + "}"; + private String json = + "{\n" + + " \"dataSetId\": 1,\n" + + " \"llmReq\": {\n" + + " \"queryText\": \"xxx2024年播放量最高的十首歌\",\n" + + " \"schema\": {\n" + + " \"dataSetName\": \"歌曲\",\n" + + " \"fieldNameList\": [\n" + + " \"商务组\",\n" + + " \"歌曲名\",\n" + + " \"播放量\",\n" + + " \"播放份额\",\n" + + " \"数据日期\"\n" + + " ]\n" + + " },\n" + + " \"linking\": [\n" + + "\n" + + " ],\n" + + " \"currentDate\": \"2024-02-24\",\n" + + " \"priorExts\": \"播放份额是小数; \",\n" + + " \"sqlGenType\": \"1_pass_self_consistency\"\n" + + " },\n" + + " \"request\": null,\n" + + " \"linkingValues\": [\n" + + "\n" + + " ]\n" + + "}"; @Test void doCorrect() throws JsonProcessingException { @@ -56,9 +57,9 @@ class SchemaCorrectorTest { ObjectMapper objectMapper = new ObjectMapper(); ParseResult parseResult = objectMapper.readValue(json, ParseResult.class); - - String sql = "select 歌曲名 from 歌曲 where 发行日期 >= '2024-01-01' " - + "and 商务组 = 'xxx' order by 播放量 desc limit 10"; + String sql = + "select 歌曲名 from 歌曲 where 发行日期 >= '2024-01-01' " + + "and 商务组 = 'xxx' order by 播放量 desc limit 10"; SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); SqlInfo sqlInfo = new SqlInfo(); sqlInfo.setParsedS2SQL(sql); @@ -69,14 +70,14 @@ class SchemaCorrectorTest { schemaElement.setDataSetId(dataSetId); semanticParseInfo.setDataSet(schemaElement); - semanticParseInfo.getProperties().put(Constants.CONTEXT, parseResult); SchemaCorrector schemaCorrector = new SchemaCorrector(); schemaCorrector.removeFilterIfNotInLinkingValue(chatQueryContext, semanticParseInfo); - Assert.assertEquals("SELECT 歌曲名 FROM 歌曲 WHERE 发行日期 >= '2024-01-01' " - + "ORDER BY 播放量 DESC LIMIT 10", semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); + Assert.assertEquals( + "SELECT 歌曲名 FROM 歌曲 WHERE 发行日期 >= '2024-01-01' " + "ORDER BY 播放量 DESC LIMIT 10", + semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); parseResult = objectMapper.readValue(json, ParseResult.class); @@ -91,9 +92,10 @@ class SchemaCorrectorTest { semanticParseInfo.getSqlInfo().setCorrectedS2SQL(sql); semanticParseInfo.getSqlInfo().setParsedS2SQL(sql); schemaCorrector.removeFilterIfNotInLinkingValue(chatQueryContext, semanticParseInfo); - Assert.assertEquals("SELECT 歌曲名 FROM 歌曲 WHERE 发行日期 >= '2024-01-01' " - + "AND 商务组 = 'xxx' ORDER BY 播放量 DESC LIMIT 10", semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); - + Assert.assertEquals( + "SELECT 歌曲名 FROM 歌曲 WHERE 发行日期 >= '2024-01-01' " + + "AND 商务组 = 'xxx' ORDER BY 播放量 DESC LIMIT 10", + semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); } private ChatQueryContext buildQueryContext(Long dataSetId) { @@ -128,4 +130,4 @@ class SchemaCorrectorTest { chatQueryContext.setSemanticSchema(semanticSchema); return chatQueryContext; } -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java index 8736ba989..d0dee8c00 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java @@ -48,7 +48,8 @@ class SelectCorrectorTest { sqlInfo.setCorrectedS2SQL(sql); semanticParseInfo.setSqlInfo(sqlInfo); corrector.correct(chatQueryContext, semanticParseInfo); - Assert.assertEquals("SELECT 粉丝数, 国籍, 艺人名, 性别 FROM 艺人库 WHERE 艺人名 = '周杰伦'", + Assert.assertEquals( + "SELECT 粉丝数, 国籍, 艺人名, 性别 FROM 艺人库 WHERE 艺人名 = '周杰伦'", semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); } @@ -111,4 +112,4 @@ class SelectCorrectorTest { chatQueryContext.setSemanticSchema(semanticSchema); return chatQueryContext; } -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrectorTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrectorTest.java index b5be7a633..ecccd9ced 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrectorTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrectorTest.java @@ -16,9 +16,10 @@ class TimeCorrectorTest { ChatQueryContext chatQueryContext = new ChatQueryContext(); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); SqlInfo sqlInfo = new SqlInfo(); - //1.数据日期 <= - String sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + // 1.数据日期 <= + String sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); semanticParseInfo.setSqlInfo(sqlInfo); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -28,9 +29,10 @@ class TimeCorrectorTest { + "AND 数据日期 >= '2023-11-17' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //2.数据日期 < - sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; + // 2.数据日期 < + sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -39,9 +41,10 @@ class TimeCorrectorTest { + "AND 数据日期 >= '2023-11-17' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //3.数据日期 >= - sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; + // 3.数据日期 >= + sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -50,9 +53,10 @@ class TimeCorrectorTest { + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //4.数据日期 > - sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; + // 4.数据日期 > + sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -61,9 +65,8 @@ class TimeCorrectorTest { + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //5.no 数据日期 - sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE 歌手名 = '张三' GROUP BY 维度1"; + // 5.no 数据日期 + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + "WHERE 歌手名 = '张三' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -71,9 +74,10 @@ class TimeCorrectorTest { "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE 歌手名 = '张三' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //6. 数据日期-月 <= - sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE 歌手名 = '张三' AND 数据日期_月 <= '2024-01' GROUP BY 维度1"; + // 6. 数据日期-月 <= + sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE 歌手名 = '张三' AND 数据日期_月 <= '2024-01' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -82,9 +86,10 @@ class TimeCorrectorTest { + "AND 数据日期_月 >= '2024-01' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //7. 数据日期-月 > - sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE 歌手名 = '张三' AND 数据日期_月 > '2024-01' GROUP BY 维度1"; + // 7. 数据日期-月 > + sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE 歌手名 = '张三' AND 数据日期_月 > '2024-01' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); @@ -93,7 +98,7 @@ class TimeCorrectorTest { + "WHERE 歌手名 = '张三' AND 数据日期_月 > '2024-01' GROUP BY 维度1", sqlInfo.getCorrectedS2SQL()); - //8. no where + // 8. no where sql = "SELECT COUNT(1) FROM 数据库"; sqlInfo.setCorrectedS2SQL(sql); corrector.doCorrect(chatQueryContext, semanticParseInfo); diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrectorTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrectorTest.java index 904b4325e..ab8593f7a 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrectorTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/WhereCorrectorTest.java @@ -10,15 +10,15 @@ import com.tencent.supersonic.headless.chat.ChatQueryContext; import org.junit.Assert; import org.junit.jupiter.api.Test; - class WhereCorrectorTest { @Test void addQueryFilter() { SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); SqlInfo sqlInfo = new SqlInfo(); - String sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " - + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + String sql = + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; sqlInfo.setCorrectedS2SQL(sql); semanticParseInfo.setSqlInfo(sqlInfo); @@ -56,8 +56,10 @@ class WhereCorrectorTest { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); - Assert.assertEquals(correctS2SQL, "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE " - + "(歌手名 = '张三') AND 数据日期 <= '2023-11-17' AND age > 30 AND " - + "name LIKE 'John%' AND id IN (1, 2, 3, 4) AND status GROUP BY 维度1"); + Assert.assertEquals( + correctS2SQL, + "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE " + + "(歌手名 = '张三') AND 数据日期 <= '2023-11-17' AND age > 30 AND " + + "name LIKE 'John%' AND id IN (1, 2, 3, 4) AND status GROUP BY 维度1"); } -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/mapper/LoadRemoveServiceTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/mapper/LoadRemoveServiceTest.java index d4ea9dc2c..7a0439916 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/mapper/LoadRemoveServiceTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/mapper/LoadRemoveServiceTest.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.mapper; - import com.hankcs.hanlp.algorithm.EditDistance; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/parser/AggregateTypeParserTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/parser/AggregateTypeParserTest.java index 383a42096..4d2a5fb87 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/parser/AggregateTypeParserTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/parser/AggregateTypeParserTest.java @@ -10,7 +10,8 @@ class AggregateTypeParserTest { @Test void getAggregateParser() { AggregateTypeParser aggregateParser = new AggregateTypeParser(); - AggregateTypeEnum aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最大值"); + AggregateTypeEnum aggregateType = + aggregateParser.resolveAggregateType("supsersonic产品访问次数最大值"); Assert.assertEquals(aggregateType, AggregateTypeEnum.MAX); aggregateType = aggregateParser.resolveAggregateType("supsersonic产品pv"); @@ -34,4 +35,4 @@ class AggregateTypeParserTest { aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数汇总"); Assert.assertEquals(aggregateType, AggregateTypeEnum.SUM); } -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMResponseServiceTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMResponseServiceTest.java index cb633663b..dd658bfb9 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMResponseServiceTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMResponseServiceTest.java @@ -23,7 +23,8 @@ class LLMResponseServiceTest { llmResp.setSqlRespMap(sqlWeight); LLMResponseService llmResponseService = new LLMResponseService(); - Map deduplicationSqlResp = llmResponseService.getDeduplicationSqlResp(0, llmResp); + Map deduplicationSqlResp = + llmResponseService.getDeduplicationSqlResp(0, llmResp); Assert.assertEquals(deduplicationSqlResp.size(), 1); @@ -51,6 +52,5 @@ class LLMResponseServiceTest { deduplicationSqlResp = llmResponseService.getDeduplicationSqlResp(0, llmResp3); Assert.assertEquals(deduplicationSqlResp.size(), 2); - } -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMSqlParserTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMSqlParserTest.java index 260f5e032..ffda22320 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMSqlParserTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/s2sql/LLMSqlParserTest.java @@ -26,31 +26,26 @@ class LLMSqlParserTest { value1.setAlias(Arrays.asList("周杰倫", "Jay Chou", "周董", "周先生")); schemaValueMaps.add(value1); - SchemaElement schemaElement = SchemaElement.builder() - .bizName("singer_name") - .name("歌手名") - .dataSetId(2L) - .schemaValueMaps(schemaValueMaps) - .build(); + SchemaElement schemaElement = + SchemaElement.builder() + .bizName("singer_name") + .name("歌手名") + .dataSetId(2L) + .schemaValueMaps(schemaValueMaps) + .build(); dimensions.add(schemaElement); - SchemaElement schemaElement2 = SchemaElement.builder() - .bizName("publish_time") - .name("发布时间") - .dataSetId(2L) - .build(); + SchemaElement schemaElement2 = + SchemaElement.builder().bizName("publish_time").name("发布时间").dataSetId(2L).build(); dimensions.add(schemaElement2); when(mockSemanticSchema.getDimensions()).thenReturn(dimensions); List metrics = new ArrayList<>(); - SchemaElement metric = SchemaElement.builder() - .bizName("play_count") - .name("播放量") - .dataSetId(2L) - .build(); + SchemaElement metric = + SchemaElement.builder().bizName("play_count").name("播放量").dataSetId(2L).build(); metrics.add(metric); when(mockSemanticSchema.getMetrics()).thenReturn(metrics); } -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParserTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParserTest.java index afb6c0f73..5e4c0aee6 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParserTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/QueryFilterParserTest.java @@ -7,7 +7,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import org.junit.Assert; import org.junit.jupiter.api.Test; - class QueryFilterParserTest { @Test @@ -41,8 +40,9 @@ class QueryFilterParserTest { String parse = QueryFilterParser.parse(queryFilters); - Assert.assertEquals(parse, "age > 30 AND name LIKE 'John%' AND id IN (1, 2, 3, 4)" - + " AND status NOT_IN ('inactive', 'deleted')"); + Assert.assertEquals( + parse, + "age > 30 AND name LIKE 'John%' AND id IN (1, 2, 3, 4)" + + " AND status NOT_IN ('inactive', 'deleted')"); } - -} \ No newline at end of file +} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/S2SqlDateHelperTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/S2SqlDateHelperTest.java index d37af5a0f..7f4a3d5f0 100644 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/S2SqlDateHelperTest.java +++ b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/utils/S2SqlDateHelperTest.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.chat.utils; - import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeMode; @@ -34,7 +33,8 @@ class S2SqlDateHelperTest { referenceDate = S2SqlDateHelper.getReferenceDate(chatQueryContext, dataSetId); Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(0)); - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); QueryConfig queryConfig = dataSetSchema.getQueryConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); timeDefaultConfig.setTimeMode(TimeMode.LAST); @@ -59,15 +59,18 @@ class S2SqlDateHelperTest { Long dataSetId = 1L; ChatQueryContext chatQueryContext = buildQueryContext(dataSetId); - Pair startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, null, QueryType.DETAIL); + Pair startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, null, QueryType.DETAIL); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(0)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(0)); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.DETAIL); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.DETAIL); Assert.assertNotNull(startEndDate.getLeft()); Assert.assertNotNull(startEndDate.getRight()); - DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + DataSetSchema dataSetSchema = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); QueryConfig queryConfig = dataSetSchema.getQueryConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); timeDefaultConfig.setTimeMode(TimeMode.LAST); @@ -76,33 +79,39 @@ class S2SqlDateHelperTest { queryConfig.getTagTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); queryConfig.getMetricTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.DETAIL); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.DETAIL); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(20)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(20)); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(20)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(20)); timeDefaultConfig.setUnit(2); timeDefaultConfig.setTimeMode(TimeMode.RECENT); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(2)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(1)); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.DETAIL); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.DETAIL); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(2)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(1)); timeDefaultConfig.setUnit(-1); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); Assert.assertNull(startEndDate.getLeft()); Assert.assertNull(startEndDate.getRight()); timeDefaultConfig.setTimeMode(TimeMode.LAST); timeDefaultConfig.setPeriod(Constants.DAY); timeDefaultConfig.setUnit(5); - startEndDate = S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); + startEndDate = + S2SqlDateHelper.getStartEndDate(chatQueryContext, dataSetId, QueryType.METRIC); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(5)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(5)); } @@ -122,4 +131,4 @@ class S2SqlDateHelperTest { chatQueryContext.setSemanticSchema(semanticSchema); return chatQueryContext; } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java index e00ab1387..77a75fec7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java @@ -40,12 +40,13 @@ public abstract class BaseDbAdaptor implements DbAdaptor { return dbs; } - public List getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException { + public List getTables(ConnectInfo connectionInfo, String schemaName) + throws SQLException { List tablesAndViews = new ArrayList<>(); DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); - try (ResultSet resultSet = metaData.getTables(schemaName, schemaName, null, - new String[]{"TABLE", "VIEW"})) { + try (ResultSet resultSet = + metaData.getTables(schemaName, schemaName, null, new String[] {"TABLE", "VIEW"})) { while (resultSet.next()) { String name = resultSet.getString("TABLE_NAME"); tablesAndViews.add(name); @@ -56,7 +57,8 @@ public abstract class BaseDbAdaptor implements DbAdaptor { return tablesAndViews; } - public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException { + public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + throws SQLException { List dbColumns = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); ResultSet columns = metaData.getColumns(schemaName, schemaName, tableName, null); @@ -70,9 +72,11 @@ public abstract class BaseDbAdaptor implements DbAdaptor { } protected DatabaseMetaData getDatabaseMetaData(ConnectInfo connectionInfo) throws SQLException { - Connection connection = DriverManager.getConnection(connectionInfo.getUrl(), - connectionInfo.getUserName(), connectionInfo.getPassword()); + Connection connection = + DriverManager.getConnection( + connectionInfo.getUrl(), + connectionInfo.getUserName(), + connectionInfo.getPassword()); return connection.getMetaData(); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java index fc153cb62..6249caf0b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java @@ -13,9 +13,11 @@ public class ClickHouseAdaptor extends BaseDbAdaptor { public String getDateFormat(String dateType, String dateFormat, String column) { if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) { if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { - return "toYYYYMM(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s", column); + return "toYYYYMM(toDate(parseDateTimeBestEffort(toString(%s))))" + .replace("%s", column); } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { - return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s", column); + return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))" + .replace("%s", column); } else { return "toDate(parseDateTimeBestEffort(toString(%s)))".replace("%s", column); } @@ -39,5 +41,4 @@ public class ClickHouseAdaptor extends BaseDbAdaptor { functionMap.put("YEAR".toLowerCase(), "toYear"); return SqlReplaceHelper.replaceFunction(sql, functionMap); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java index b81bb9ed6..63a478e7f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java @@ -2,12 +2,11 @@ package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.core.pojo.ConnectInfo; + import java.sql.SQLException; import java.util.List; -/** - * Adapters for different query engines to obtain table, field, and time formatting methods - */ +/** Adapters for different query engines to obtain table, field, and time formatting methods */ public interface DbAdaptor { String getDateFormat(String dateType, String dateFormat, String column); @@ -18,6 +17,6 @@ public interface DbAdaptor { List getTables(ConnectInfo connectInfo, String schemaName) throws SQLException; - List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException; - + List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + throws SQLException; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java index 15e62ffb9..0588b0b0a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java @@ -5,7 +5,6 @@ import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import java.util.HashMap; import java.util.Map; - public class DbAdaptorFactory { private static Map dbAdaptorMap; @@ -22,5 +21,4 @@ public class DbAdaptorFactory { public static DbAdaptor getEngineAdaptor(String engineType) { return dbAdaptorMap.get(engineType); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java index 18c434a06..ae7caae10 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.adaptor.db; - public class DefaultDbAdaptor extends BaseDbAdaptor { @Override @@ -12,5 +11,4 @@ public class DefaultDbAdaptor extends BaseDbAdaptor { public String functionNameCorrector(String sql) { return sql; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java index 687eb96d7..9c367972b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java @@ -9,15 +9,18 @@ public class H2Adaptor extends BaseDbAdaptor { public String getDateFormat(String dateType, String dateFormat, String column) { if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) { if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { - return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM')".replace("%s", column); + return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM')" + .replace("%s", column); } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { return "DATE_TRUNC('week',%s)".replace("%s", column); } else { - return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM-dd')".replace("%s", column); + return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM-dd')" + .replace("%s", column); } } else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) { if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { - return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyy-MM-dd'),'yyyy-MM') ".replace("%s", column); + return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyy-MM-dd'),'yyyy-MM') " + .replace("%s", column); } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { return "DATE_TRUNC('week',%s)".replace("%s", column); } else { @@ -31,5 +34,4 @@ public class H2Adaptor extends BaseDbAdaptor { public String functionNameCorrector(String sql) { return sql; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java index b77cdb309..64e3a5a4c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java @@ -1,21 +1,19 @@ package com.tencent.supersonic.headless.core.adaptor.db; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.Constants; - +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; public class MysqlAdaptor extends BaseDbAdaptor { - /** - * transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY) - */ + /** transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY) */ @Override public String getDateFormat(String dateType, String dateFormat, String column) { if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) { if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { return "DATE_FORMAT(%s, '%Y-%m')".replace("%s", column); } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { - return "DATE_FORMAT(DATE_SUB(%s, INTERVAL (DAYOFWEEK(%s) - 2) DAY), '%Y-%m-%d')".replace("%s", column); + return "DATE_FORMAT(DATE_SUB(%s, INTERVAL (DAYOFWEEK(%s) - 2) DAY), '%Y-%m-%d')" + .replace("%s", column); } else { return "date_format(str_to_date(%s, '%Y%m%d'),'%Y-%m-%d')".replace("%s", column); } @@ -23,7 +21,8 @@ public class MysqlAdaptor extends BaseDbAdaptor { if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { return "DATE_FORMAT(%s, '%Y-%m') ".replace("%s", column); } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { - return "DATE_FORMAT(DATE_SUB(%s, INTERVAL (DAYOFWEEK(%s) - 2) DAY), '%Y-%m-%d')".replace("%s", column); + return "DATE_FORMAT(DATE_SUB(%s, INTERVAL (DAYOFWEEK(%s) - 2) DAY), '%Y-%m-%d')" + .replace("%s", column); } else { return column; } @@ -35,5 +34,4 @@ public class MysqlAdaptor extends BaseDbAdaptor { public String functionNameCorrector(String sql) { return sql; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java index 71139799c..ddb29f628 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java @@ -26,9 +26,11 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { public String getDateFormat(String dateType, String dateFormat, String column) { if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) { if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { - return "formatDateTime(toDate(parseDateTimeBestEffort(toString(%s))),'%Y-%m')".replace("%s", column); + return "formatDateTime(toDate(parseDateTimeBestEffort(toString(%s))),'%Y-%m')" + .replace("%s", column); } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { - return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s", column); + return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))" + .replace("%s", column); } else { return "toDate(parseDateTimeBestEffort(toString(%s)))".replace("%s", column); } @@ -51,38 +53,45 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { functionMap.put("DAY".toLowerCase(), "TO_CHAR"); functionMap.put("YEAR".toLowerCase(), "TO_CHAR"); Map functionCall = new HashMap<>(); - functionCall.put("MONTH".toLowerCase(), o -> { - if (Objects.nonNull(o) && o instanceof ExpressionList) { - ExpressionList expressionList = (ExpressionList) o; - expressionList.add(new StringValue("MM")); - return expressionList; - } - return o; - }); - functionCall.put("DAY".toLowerCase(), o -> { - if (Objects.nonNull(o) && o instanceof ExpressionList) { - ExpressionList expressionList = (ExpressionList) o; - expressionList.add(new StringValue("dd")); - return expressionList; - } - return o; - }); - functionCall.put("YEAR".toLowerCase(), o -> { - if (Objects.nonNull(o) && o instanceof ExpressionList) { - ExpressionList expressionList = (ExpressionList) o; - expressionList.add(new StringValue("YYYY")); - return expressionList; - } - return o; - }); + functionCall.put( + "MONTH".toLowerCase(), + o -> { + if (Objects.nonNull(o) && o instanceof ExpressionList) { + ExpressionList expressionList = (ExpressionList) o; + expressionList.add(new StringValue("MM")); + return expressionList; + } + return o; + }); + functionCall.put( + "DAY".toLowerCase(), + o -> { + if (Objects.nonNull(o) && o instanceof ExpressionList) { + ExpressionList expressionList = (ExpressionList) o; + expressionList.add(new StringValue("dd")); + return expressionList; + } + return o; + }); + functionCall.put( + "YEAR".toLowerCase(), + o -> { + if (Objects.nonNull(o) && o instanceof ExpressionList) { + ExpressionList expressionList = (ExpressionList) o; + expressionList.add(new StringValue("YYYY")); + return expressionList; + } + return o; + }); return SqlReplaceHelper.replaceFunction(sql, functionMap, functionCall); } - public List getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException { + public List getTables(ConnectInfo connectionInfo, String schemaName) + throws SQLException { List tablesAndViews = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); - try (ResultSet resultSet = metaData.getTables(null, null, null, - new String[]{"TABLE", "VIEW"})) { + try (ResultSet resultSet = + metaData.getTables(null, null, null, new String[] {"TABLE", "VIEW"})) { while (resultSet.next()) { String name = resultSet.getString("TABLE_NAME"); tablesAndViews.add(name); @@ -93,7 +102,8 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { return tablesAndViews; } - public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException { + public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + throws SQLException { List dbColumns = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); ResultSet columns = metaData.getColumns(null, null, tableName, null); @@ -105,5 +115,4 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { } return dbColumns; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java index 7f936c157..266b17a4c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java @@ -22,6 +22,4 @@ public class CacheCommonConfig { @Value("${s2.query.cache.enable:true}") private Boolean cacheEnable; - - -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheManager.java index 4bb488bcb..5678c95c6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheManager.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.cache; - public interface CacheManager { Boolean put(String key, Object value); @@ -10,5 +9,4 @@ public interface CacheManager { String generateCacheKey(String prefix, String body); Boolean removeCache(String key); - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheConfig.java index 439002674..1e9ac6220 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheConfig.java @@ -12,8 +12,7 @@ import java.util.concurrent.TimeUnit; @Configuration public class CaffeineCacheConfig { - @Autowired - private CacheCommonConfig cacheCommonConfig; + @Autowired private CacheCommonConfig cacheCommonConfig; @Value("${s2.caffeine.initial.capacity:500}") private Integer caffeineInitialCapacity; @@ -24,7 +23,8 @@ public class CaffeineCacheConfig { @Bean(name = "caffeineCache") public Cache caffeineCache() { return Caffeine.newBuilder() - .expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES) + .expireAfterWrite( + cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES) .initialCapacity(caffeineInitialCapacity) .maximumSize(caffeineMaximumSize) .build(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheManager.java index d9cdc0dae..b7c53fa5b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheManager.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.cache; - import com.github.benmanes.caffeine.cache.Cache; import com.google.common.base.Joiner; import lombok.extern.slf4j.Slf4j; @@ -13,8 +12,7 @@ import org.springframework.stereotype.Component; @Slf4j public class CaffeineCacheManager implements CacheManager { - @Autowired - private CacheCommonConfig cacheCommonConfig; + @Autowired private CacheCommonConfig cacheCommonConfig; @Autowired @Qualifier("caffeineCache") @@ -39,8 +37,13 @@ public class CaffeineCacheManager implements CacheManager { if (StringUtils.isEmpty(prefix)) { prefix = "-1"; } - return Joiner.on(":").join(cacheCommonConfig.getCacheCommonApp(), cacheCommonConfig.getCacheCommonEnv(), - cacheCommonConfig.getCacheCommonVersion(), prefix, body); + return Joiner.on(":") + .join( + cacheCommonConfig.getCacheCommonApp(), + cacheCommonConfig.getCacheCommonEnv(), + cacheCommonConfig.getCacheCommonVersion(), + prefix, + body); } @Override diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java index 328642487..c8593d7a3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.cache; - import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import lombok.extern.slf4j.Slf4j; @@ -30,10 +29,11 @@ public class DefaultQueryCache implements QueryCache { CacheCommonConfig cacheCommonConfig = ContextUtils.getBean(CacheCommonConfig.class); if (cacheCommonConfig.getCacheEnable() && Objects.nonNull(value)) { CompletableFuture.supplyAsync(() -> cacheManager.put(cacheKey, value)) - .exceptionally(exception -> { - log.warn("exception:", exception); - return null; - }); + .exceptionally( + exception -> { + log.warn("exception:", exception); + return null; + }); log.debug("put to cache, key: {}", cacheKey); return true; } @@ -48,7 +48,8 @@ public class DefaultQueryCache implements QueryCache { } private String getKeyByModelIds(List modelIds) { - return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + return String.join( + ",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); } private boolean isCache(SemanticQueryReq semanticQueryReq) { @@ -61,5 +62,4 @@ public class DefaultQueryCache implements QueryCache { } return false; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java index 8fe71782e..4dc7cdb1d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.cache; - import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; public interface QueryCache { @@ -10,5 +9,4 @@ public interface QueryCache { Boolean put(String cacheKey, Object value); String getCacheKey(SemanticQueryReq semanticQueryReq); - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java index 9fdfd7368..60e57ded4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java @@ -3,11 +3,10 @@ package com.tencent.supersonic.headless.core.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; + @Data @Configuration public class AggregatorConfig { @Value("${s2.metric.aggregator.ratio.enable:true}") private Boolean enableRatio; } - - diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java index 408af65c9..4fb51c2fc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java @@ -9,20 +9,13 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class DefaultMetric { - /** - * default metrics - */ + /** default metrics */ private Long metricId; - /** - * default time span unit - */ + /** default time span unit */ private Integer unit; - /** - * default time type: DAY - * DAY, WEEK, MONTH, YEAR - */ + /** default time type: DAY DAY, WEEK, MONTH, YEAR */ private String period; private String bizName; @@ -39,4 +32,4 @@ public class DefaultMetric { this.unit = unit; this.period = period; } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java index 3c7db1fe4..262d58b62 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java @@ -4,29 +4,17 @@ import com.tencent.supersonic.common.pojo.Constants; import lombok.Data; import lombok.ToString; - -/** - * default metrics about the model - */ - +/** default metrics about the model */ @ToString @Data public class DefaultMetricInfo { - /** - * default metrics - */ + /** default metrics */ private Long metricId; - /** - * default time span unit - */ + /** default time span unit */ private Integer unit = 1; - /** - * default time type: day - * DAY, WEEK, MONTH, YEAR - */ + /** default time type: day DAY, WEEK, MONTH, YEAR */ private String period = Constants.DAY; - -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java index 31487d3ab..e9bd9d60e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java @@ -40,5 +40,4 @@ public class DefaultSemanticConfig { @Value("${s2.explain.path:/api/semantic/query/explain}") private String explainPath; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java index ee35b5a60..eefbb1032 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java @@ -18,5 +18,4 @@ public class Dim4Dict { private List blackList; private List whiteList; private List ruleList; - -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java index d485cf2fb..a89c64f0c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java @@ -4,13 +4,10 @@ import lombok.Data; import java.util.List; -/** - * when query an entity, return related dimension/metric info - */ +/** when query an entity, return related dimension/metric info */ @Data public class EntityDetailData { private List dimensionIds; private List metricIds; - -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java index ce9d4dceb..d741551f4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.config; - import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import lombok.Data; @@ -12,4 +11,4 @@ public class EntityInternalDetail { List dimensionList; List metricList; -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ExecutorConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ExecutorConfig.java index b34407057..909a1d78c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ExecutorConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ExecutorConfig.java @@ -10,8 +10,10 @@ public class ExecutorConfig { @Value("${s2.metricParser.agg.mysql.lowVersion:5.7}") private String mysqlLowVersion; + @Value("${s2.metricParser.agg.ck.lowVersion:20.4}") private String ckLowVersion; + @Value("${s2.internal.metric.cnt.suffix:internal_cnt}") private String internalMetricNameSuffix; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java index a6840ac1f..92cf34429 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java @@ -1,20 +1,12 @@ package com.tencent.supersonic.headless.core.executor; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.core.pojo.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.Configuration; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.TimeRange; import com.tencent.supersonic.headless.core.translator.calcite.schema.DataSourceTable; import com.tencent.supersonic.headless.core.translator.calcite.schema.DataSourceTable.Builder; import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; -import com.tencent.supersonic.headless.core.pojo.Materialization; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.config.CalciteConnectionConfigImpl; @@ -45,9 +37,16 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.springframework.util.CollectionUtils; -/** - * abstract of accelerator , provide Basic methods - */ +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +/** abstract of accelerator , provide Basic methods */ @Slf4j public abstract class AbstractAccelerator implements QueryAccelerator { @@ -56,11 +55,13 @@ public abstract class AbstractAccelerator implements QueryAccelerator { public static final String MATERIALIZATION_SYS_VIEW = "sys_view"; public static final String MATERIALIZATION_SYS_PARTITION = "sys_partition"; - /** - * check if a materialization match the fields and partitions - */ - protected boolean check(RelOptPlanner relOptPlanner, RelBuilder relBuilder, - CalciteCatalogReader calciteCatalogReader, Materialization materialization, List fields, + /** check if a materialization match the fields and partitions */ + protected boolean check( + RelOptPlanner relOptPlanner, + RelBuilder relBuilder, + CalciteCatalogReader calciteCatalogReader, + Materialization materialization, + List fields, List> partitions) { if (!materialization.isPartitioned()) { return fields.stream().allMatch(f -> materialization.getColumns().contains(f)); @@ -75,7 +76,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator { Set materializationFields = new HashSet<>(viewFields); materializationFields.addAll(queryFields); - List materializationFieldList = materializationFields.stream().collect(Collectors.toList()); + List materializationFieldList = + materializationFields.stream().collect(Collectors.toList()); relBuilder.clear(); if (!CollectionUtils.isEmpty(relOptPlanner.getMaterializations())) { @@ -83,32 +85,41 @@ public abstract class AbstractAccelerator implements QueryAccelerator { } Materialization viewMaterialization = Materialization.builder().build(); - viewMaterialization.setName(String.format("%s.%s", MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW)); + viewMaterialization.setName( + String.format("%s.%s", MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW)); viewMaterialization.setColumns(viewFieldList); addMaterialization(calciteCatalogReader.getRootSchema(), viewMaterialization); Materialization queryMaterialization = Materialization.builder().build(); - queryMaterialization.setName(String.format("%s.%s", MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE)); + queryMaterialization.setName( + String.format("%s.%s", MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE)); queryMaterialization.setColumns(materializationFieldList); addMaterialization(calciteCatalogReader.getRootSchema(), queryMaterialization); - RelNode replacement = relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW)).build(); - RelBuilder viewBuilder = relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE)); + RelNode replacement = + relBuilder + .scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW)) + .build(); + RelBuilder viewBuilder = + relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE)); if (materialization.isPartitioned()) { - RexNode viewFilter = getRexNode(relBuilder, materialization, - MATERIALIZATION_SYS_PARTITION); + RexNode viewFilter = + getRexNode(relBuilder, materialization, MATERIALIZATION_SYS_PARTITION); viewBuilder = viewBuilder.filter(viewFilter); } RelNode viewRel = project(viewBuilder, viewFieldList).build(); List view = Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW); - RelOptMaterialization relOptMaterialization = new RelOptMaterialization(replacement, viewRel, null, - view); + RelOptMaterialization relOptMaterialization = + new RelOptMaterialization(replacement, viewRel, null, view); relOptPlanner.addMaterialization(relOptMaterialization); - RelBuilder checkBuilder = relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE)); + RelBuilder checkBuilder = + relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE)); if (materialization.isPartitioned()) { - checkBuilder = checkBuilder.filter(getRexNode(checkBuilder, partitions, MATERIALIZATION_SYS_PARTITION)); + checkBuilder = + checkBuilder.filter( + getRexNode(checkBuilder, partitions, MATERIALIZATION_SYS_PARTITION)); } RelNode checkRel = project(checkBuilder, queryFieldList).build(); relOptPlanner.setRoot(checkRel); @@ -124,11 +135,12 @@ public abstract class AbstractAccelerator implements QueryAccelerator { protected CalciteCatalogReader getCalciteCatalogReader() { CalciteCatalogReader calciteCatalogReader; CalciteSchema viewSchema = SchemaBuilder.getMaterializationSchema(); - calciteCatalogReader = new CalciteCatalogReader( - CalciteSchema.from(viewSchema.plus()), - CalciteSchema.from(viewSchema.plus()).path(null), - Configuration.typeFactory, - new CalciteConnectionConfigImpl(new Properties())); + calciteCatalogReader = + new CalciteCatalogReader( + CalciteSchema.from(viewSchema.plus()), + CalciteSchema.from(viewSchema.plus()).path(null), + Configuration.typeFactory, + new CalciteConnectionConfigImpl(new Properties())); return calciteCatalogReader; } @@ -139,8 +151,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator { return relOptPlanner; } - protected RelBuilder builderMaterializationPlan(CalciteCatalogReader calciteCatalogReader, - RelOptPlanner relOptPlanner) { + protected RelBuilder builderMaterializationPlan( + CalciteCatalogReader calciteCatalogReader, RelOptPlanner relOptPlanner) { relOptPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); relOptPlanner.addRelTraitDef(RelDistributionTraitDef.INSTANCE); EnumerableRules.rules().forEach(relOptPlanner::addRule); @@ -149,7 +161,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator { return RelFactories.LOGICAL_BUILDER.create(relOptCluster, calciteCatalogReader); } - protected void addMaterialization(CalciteSchema dataSetSchema, Materialization materialization) { + protected void addMaterialization( + CalciteSchema dataSetSchema, Materialization materialization) { String[] dbTable = materialization.getName().split("\\."); String tb = dbTable[1].toLowerCase(); String db = dbTable[0].toLowerCase(); @@ -171,31 +184,38 @@ public abstract class AbstractAccelerator implements QueryAccelerator { } else { dataSetSchema.add(tb, srcTable); } - } protected Set extractTableNames(RelNode relNode) { Set tableNames = new HashSet<>(); - RelShuttle shuttle = new RelHomogeneousShuttle() { - public RelNode visit(TableScan scan) { - RelOptTable table = scan.getTable(); - tableNames.addAll(table.getQualifiedName()); - return scan; - } - }; + RelShuttle shuttle = + new RelHomogeneousShuttle() { + public RelNode visit(TableScan scan) { + RelOptTable table = scan.getTable(); + tableNames.addAll(table.getQualifiedName()); + return scan; + } + }; relNode.accept(shuttle); return tableNames; } - protected RexNode getRexNodeByTimeRange(RelBuilder relBuilder, TimeRange timeRange, String field) { - return relBuilder.call(SqlStdOperatorTable.AND, - relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, relBuilder.field(field), + protected RexNode getRexNodeByTimeRange( + RelBuilder relBuilder, TimeRange timeRange, String field) { + return relBuilder.call( + SqlStdOperatorTable.AND, + relBuilder.call( + SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + relBuilder.field(field), relBuilder.literal(timeRange.getStart())), - relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(field), + relBuilder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + relBuilder.field(field), relBuilder.literal(timeRange.getEnd()))); } - protected RexNode getRexNode(RelBuilder relBuilder, Materialization materialization, String viewField) { + protected RexNode getRexNode( + RelBuilder relBuilder, Materialization materialization, String viewField) { RexNode rexNode = null; for (String partition : materialization.getPartitions()) { TimeRange timeRange = TimeRange.builder().start(partition).end(partition).build(); @@ -203,32 +223,50 @@ public abstract class AbstractAccelerator implements QueryAccelerator { rexNode = getRexNodeByTimeRange(relBuilder, timeRange, viewField); continue; } - rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode, - getRexNodeByTimeRange(relBuilder, timeRange, viewField)); + rexNode = + relBuilder.call( + SqlStdOperatorTable.OR, + rexNode, + getRexNodeByTimeRange(relBuilder, timeRange, viewField)); } return rexNode; } - protected RexNode getRexNode(RelBuilder relBuilder, List> timeRanges, + protected RexNode getRexNode( + RelBuilder relBuilder, + List> timeRanges, String viewField) { RexNode rexNode = null; for (ImmutablePair timeRange : timeRanges) { if (rexNode == null) { - rexNode = getRexNodeByTimeRange(relBuilder, - TimeRange.builder().start(timeRange.left).end(timeRange.right).build(), - viewField); + rexNode = + getRexNodeByTimeRange( + relBuilder, + TimeRange.builder() + .start(timeRange.left) + .end(timeRange.right) + .build(), + viewField); continue; } - rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode, - getRexNodeByTimeRange(relBuilder, - TimeRange.builder().start(timeRange.left).end(timeRange.right).build(), - viewField)); + rexNode = + relBuilder.call( + SqlStdOperatorTable.OR, + rexNode, + getRexNodeByTimeRange( + relBuilder, + TimeRange.builder() + .start(timeRange.left) + .end(timeRange.right) + .build(), + viewField)); } return rexNode; } private static RelBuilder project(RelBuilder relBuilder, List fields) { - List rexNodes = fields.stream().map(f -> relBuilder.field(f)).collect(Collectors.toList()); + List rexNodes = + fields.stream().map(f -> relBuilder.field(f)).collect(Collectors.toList()); return relBuilder.project(rexNodes); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index da629ca2a..42f3d5fe6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -26,8 +26,10 @@ public class JdbcExecutor implements QueryExecutor { for (QueryAccelerator queryAccelerator : ComponentFactory.getQueryAccelerators()) { if (queryAccelerator.check(queryStatement)) { SemanticQueryResp semanticQueryResp = queryAccelerator.query(queryStatement); - if (Objects.nonNull(semanticQueryResp) && !semanticQueryResp.getResultList().isEmpty()) { - log.info("query by Accelerator {}", queryAccelerator.getClass().getSimpleName()); + if (Objects.nonNull(semanticQueryResp) + && !semanticQueryResp.getResultList().isEmpty()) { + log.info( + "query by Accelerator {}", queryAccelerator.getClass().getSimpleName()); return semanticQueryResp; } } @@ -43,5 +45,4 @@ public class JdbcExecutor implements QueryExecutor { queryResultWithColumns.setSql(sql); return queryResultWithColumns; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryAccelerator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryAccelerator.java index 9556445dc..0f273ccb8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryAccelerator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryAccelerator.java @@ -4,8 +4,8 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** - * customize various query media ( like duckDb redis) to improved query performance - * check ok and query successful , return SemanticQueryResp to interface immediately + * customize various query media ( like duckDb redis) to improved query performance check ok and + * query successful , return SemanticQueryResp to interface immediately */ public interface QueryAccelerator { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java index 23b326922..2e41c7974 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java @@ -3,9 +3,7 @@ package com.tencent.supersonic.headless.core.executor; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -/** - * QueryExecutor submits SQL to the database engine and performs acceleration if necessary. - */ +/** QueryExecutor submits SQL to the database engine and performs acceleration if necessary. */ public interface QueryExecutor { boolean accept(QueryStatement queryStatement); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java index 305865e47..1b5ef99df 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java @@ -1,12 +1,10 @@ package com.tencent.supersonic.headless.core.pojo; - import lombok.Data; @Data public class ConnectInfo { - private String url; private String userName; @@ -14,5 +12,4 @@ public class ConnectInfo { private String password; private String database; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java index 70bba222a..51a908024 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.pojo; - import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.util.AESEncryptionUtil; @@ -36,9 +35,7 @@ public class Database extends RecordInfo { private String database; private String schema; - /** - * mysql,clickhouse - */ + /** mysql,clickhouse */ private String type; private List admins = Lists.newArrayList(); @@ -48,5 +45,4 @@ public class Database extends RecordInfo { public String passwordDecrypt() { return AESEncryptionUtil.aesDecryptECB(password); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DuckDbSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DuckDbSource.java index dd01635ee..0b2ca7c6b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DuckDbSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DuckDbSource.java @@ -1,25 +1,24 @@ package com.tencent.supersonic.headless.core.pojo; +import javax.sql.DataSource; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.config.ExecutorConfig; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.sql.DataSource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Component; -/** - * duckDb connection session object - */ +/** duckDb connection session object */ @Component @Slf4j public class DuckDbSource { @@ -62,8 +61,10 @@ public class DuckDbSource { } protected void init(JdbcTemplate jdbcTemplate) { - jdbcTemplate.execute(String.format("SET memory_limit = '%sGB';", executorConfig.getMemoryLimit())); - jdbcTemplate.execute(String.format("SET temp_directory='%s';", executorConfig.getDuckDbTemp())); + jdbcTemplate.execute( + String.format("SET memory_limit = '%sGB';", executorConfig.getMemoryLimit())); + jdbcTemplate.execute( + String.format("SET temp_directory='%s';", executorConfig.getDuckDbTemp())); jdbcTemplate.execute(String.format("SET threads TO %s;", executorConfig.getThreads())); jdbcTemplate.execute("SET enable_object_cache = true;"); } @@ -81,21 +82,23 @@ public class DuckDbSource { } public void query(String sql, SemanticQueryResp queryResultWithColumns) { - duckDbJdbcTemplate.query(sql, rs -> { - if (null == rs) { - return queryResultWithColumns; - } - ResultSetMetaData metaData = rs.getMetaData(); - List queryColumns = new ArrayList<>(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String key = metaData.getColumnLabel(i); - queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i))); - } - queryResultWithColumns.setColumns(queryColumns); - List> resultList = buildResult(rs); - queryResultWithColumns.setResultList(resultList); - return queryResultWithColumns; - }); + duckDbJdbcTemplate.query( + sql, + rs -> { + if (null == rs) { + return queryResultWithColumns; + } + ResultSetMetaData metaData = rs.getMetaData(); + List queryColumns = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String key = metaData.getColumnLabel(i); + queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i))); + } + queryResultWithColumns.setColumns(queryColumns); + List> resultList = buildResult(rs); + queryResultWithColumns.setResultList(resultList); + return queryResultWithColumns; + }); } public static List> buildResult(ResultSet resultSet) { @@ -148,7 +151,8 @@ public class DuckDbSource { row.put(column, resultSet.getObject(i)); break; default: - throw new Exception("get result row type not found :" + rsMeta.getColumnType(i)); + throw new Exception( + "get result row type not found :" + rsMeta.getColumnType(i)); } } list.add(row); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java index 0e011596a..7954cbd78 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java @@ -203,8 +203,10 @@ public class JdbcDataSource { // default validation query String driverName = druidDataSource.getDriverClassName(); - if (driverName.indexOf("sqlserver") != -1 || driverName.indexOf("mysql") != -1 - || driverName.indexOf("h2") != -1 || driverName.indexOf("moonbox") != -1) { + if (driverName.indexOf("sqlserver") != -1 + || driverName.indexOf("mysql") != -1 + || driverName.indexOf("h2") != -1 + || driverName.indexOf("moonbox") != -1) { druidDataSource.setValidationQuery("select 1"); } @@ -240,9 +242,12 @@ public class JdbcDataSource { } private String getDataSourceKey(Database database) { - return JdbcDataSourceUtils.getKey(database.getName(), + return JdbcDataSourceUtils.getKey( + database.getName(), database.getUrl(), database.getUsername(), - database.passwordDecrypt(), "", false); + database.passwordDecrypt(), + "", + false); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Materialization.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Materialization.java index 43b95d773..cb05df30f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Materialization.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Materialization.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.headless.core.pojo; -import java.util.List; import lombok.Builder; import lombok.Data; +import java.util.List; + @Data @Builder public class Materialization { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java index 77b16f91d..2a3bc8b8b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.common.pojo.ColumnOrder; import lombok.Data; + import java.util.List; @Data @@ -13,5 +14,4 @@ public class MetricQueryParam { private Long limit; private List order; private boolean nativeQuery = false; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index bb8579e30..6bd77fd0b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -54,12 +54,17 @@ public class DefaultSemanticTranslator implements SemanticTranslator { headlessConverter.convert(queryStatement); } } - log.debug("SemanticConverter after {} {} {}", queryParam, queryStatement.getDataSetQueryParam(), + log.debug( + "SemanticConverter after {} {} {}", + queryParam, + queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam()); if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) { doParse(queryStatement.getDataSetQueryParam(), queryStatement); } else { - queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); + queryStatement + .getMetricQueryParam() + .setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); doParse(queryStatement); } if (StringUtils.isEmpty(queryStatement.getSql())) { @@ -67,38 +72,56 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } if (StringUtils.isNotBlank(queryStatement.getSql()) && !SqlSelectHelper.hasLimit(queryStatement.getSql())) { - String querySql = queryStatement.getSql() + " limit " + queryStatement.getLimit().toString(); + String querySql = + queryStatement.getSql() + " limit " + queryStatement.getLimit().toString(); queryStatement.setSql(querySql); } } - public QueryStatement doParse(DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) { + public QueryStatement doParse( + DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) { log.info("parse dataSetQuery [{}] ", dataSetQueryParam); try { if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) { List tables = new ArrayList<>(); boolean isSingleTable = dataSetQueryParam.getTables().size() == 1; for (MetricTable metricTable : dataSetQueryParam.getTables()) { - QueryStatement tableSql = parserSql(metricTable, isSingleTable, - dataSetQueryParam, queryStatement); - if (isSingleTable && Objects.nonNull(tableSql.getDataSetQueryParam()) + QueryStatement tableSql = + parserSql( + metricTable, isSingleTable, dataSetQueryParam, queryStatement); + if (isSingleTable + && Objects.nonNull(tableSql.getDataSetQueryParam()) && !tableSql.getDataSetSimplifySql().isEmpty()) { queryStatement.setSql(tableSql.getDataSetSimplifySql()); queryStatement.setDataSetQueryParam(dataSetQueryParam); return queryStatement; } - tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()}); + tables.add(new String[] {metricTable.getAlias(), tableSql.getSql()}); } if (!tables.isEmpty()) { String sql; if (dataSetQueryParam.isSupportWith()) { - sql = "with " + tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect( - Collectors.joining(",")) + "\n" + dataSetQueryParam.getSql(); + sql = + "with " + + tables.stream() + .map(t -> String.format("%s as (%s)", t[0], t[1])) + .collect(Collectors.joining(",")) + + "\n" + + dataSetQueryParam.getSql(); } else { sql = dataSetQueryParam.getSql(); for (String[] tb : tables) { - sql = StringUtils.replace(sql, tb[0], - "(" + tb[1] + ") " + (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1); + sql = + StringUtils.replace( + sql, + tb[0], + "(" + + tb[1] + + ") " + + (dataSetQueryParam.isWithAlias() + ? "" + : tb[0]), + -1); } } queryStatement.setSql(sql); @@ -114,8 +137,9 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } public QueryStatement doParse(QueryStatement queryStatement) { - return doParse(queryStatement, AggOption.getAggregation( - queryStatement.getMetricQueryParam().isNativeQuery())); + return doParse( + queryStatement, + AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery())); } public QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) { @@ -130,9 +154,12 @@ public class DefaultSemanticTranslator implements SemanticTranslator { return queryStatement; } - private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, - DataSetQueryParam dataSetQueryParam, - QueryStatement queryStatement) throws Exception { + private QueryStatement parserSql( + MetricTable metricTable, + Boolean isSingleMetricTable, + DataSetQueryParam dataSetQueryParam, + QueryStatement queryStatement) + throws Exception { MetricQueryParam metricReq = new MetricQueryParam(); metricReq.setMetrics(metricTable.getMetrics()); metricReq.setDimensions(metricTable.getDimensions()); @@ -151,10 +178,11 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } tableSql = doParse(tableSql, metricTable.getAggOption()); if (!tableSql.isOk()) { - throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(), - tableSql.getErrMsg())); + throw new Exception( + String.format( + "parser table [%s] error [%s]", + metricTable.getAlias(), tableSql.getErrMsg())); } return tableSql; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java index ae679b744..e18ad0eab 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java @@ -10,9 +10,7 @@ import org.springframework.util.CollectionUtils; import java.util.Objects; import java.util.stream.Collectors; -/** - * Remove the default metric added by the system when the query only has dimensions - */ +/** Remove the default metric added by the system when the query only has dimensions */ @Slf4j @Component("DetailQueryOptimizer") public class DetailQueryOptimizer implements QueryOptimizer { @@ -26,10 +24,14 @@ public class DetailQueryOptimizer implements QueryOptimizer { } log.debug("before handleNoMetric, sql:{}", sqlRaw); if (isDetailQuery(queryParam)) { - if (queryParam.getMetrics().size() == 0 && !CollectionUtils.isEmpty(queryParam.getGroups())) { + if (queryParam.getMetrics().size() == 0 + && !CollectionUtils.isEmpty(queryParam.getGroups())) { String sqlForm = "select %s from ( %s ) src_no_metric"; - String sql = String.format(sqlForm, queryParam.getGroups().stream().collect( - Collectors.joining(",")), sqlRaw); + String sql = + String.format( + sqlForm, + queryParam.getGroups().stream().collect(Collectors.joining(",")), + sqlRaw); queryStatement.setSql(sql); } } @@ -37,7 +39,8 @@ public class DetailQueryOptimizer implements QueryOptimizer { } public boolean isDetailQuery(QueryParam queryParam) { - return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery() + return Objects.nonNull(queryParam) + && queryParam.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(queryParam.getMetrics()); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryOptimizer.java index 970104d3b..a66bb7f12 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryOptimizer.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** - * A query optimizer rewrites physical SQL by following a set of - * optimization rules, trying to derive the most efficient query. + * A query optimizer rewrites physical SQL by following a set of optimization rules, trying to + * derive the most efficient query. */ public interface QueryOptimizer { void rewrite(QueryStatement queryStatement); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java index d4ecef5be..b15212c65 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java @@ -3,9 +3,7 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -/** - * A query parser generates physical SQL for the QueryStatement. - */ +/** A query parser generates physical SQL for the QueryStatement. */ public interface QueryParser { void parse(QueryStatement queryStatement, AggOption aggOption) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java index 4e1c03510..1053eb272 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java @@ -3,11 +3,10 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** - * SemanticTranslator converts semantic query statement into SQL statement that - * can be executed against physical data models. + * SemanticTranslator converts semantic query statement into SQL statement that can be executed + * against physical data models. */ public interface SemanticTranslator { void translate(QueryStatement queryStatement); - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java index 54245adf0..069cd1d93 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java @@ -3,20 +3,18 @@ package com.tencent.supersonic.headless.core.translator.calcite; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.QueryParser; import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import java.util.Objects; -/** - * the calcite parse implements - */ +/** the calcite parse implements */ @Component("CalciteQueryParser") @Slf4j public class CalciteQueryParser implements QueryParser { @@ -33,14 +31,21 @@ public class CalciteQueryParser implements QueryParser { SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); AggPlanner aggBuilder = new AggPlanner(semanticSchema); aggBuilder.explain(queryStatement, isAgg); - EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); queryStatement.setSql(aggBuilder.getSql(engineType)); - if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize() - && Objects.nonNull(queryStatement.getDataSetAlias()) && !queryStatement.getDataSetAlias().isEmpty()) { + if (Objects.nonNull(queryStatement.getEnableOptimize()) + && queryStatement.getEnableOptimize() + && Objects.nonNull(queryStatement.getDataSetAlias()) + && !queryStatement.getDataSetAlias().isEmpty()) { // simplify model sql with query sql - String simplifySql = aggBuilder.simplify( - getSqlByDataSet(aggBuilder.getSql(engineType), queryStatement.getDataSetSql(), - queryStatement.getDataSetAlias()), engineType); + String simplifySql = + aggBuilder.simplify( + getSqlByDataSet( + aggBuilder.getSql(engineType), + queryStatement.getDataSetSql(), + queryStatement.getDataSetAlias()), + engineType); if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { log.debug("simplifySql [{}]", simplifySql); queryStatement.setDataSetSimplifySql(simplifySql); @@ -48,15 +53,20 @@ public class CalciteQueryParser implements QueryParser { } } - private SemanticSchema getSemanticSchema(SemanticModel semanticModel, QueryStatement queryStatement) { - SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build(); + private SemanticSchema getSemanticSchema( + SemanticModel semanticModel, QueryStatement queryStatement) { + SemanticSchema semanticSchema = + SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build(); semanticSchema.setSemanticModel(semanticModel); semanticSchema.setDatasource(semanticModel.getDatasourceMap()); semanticSchema.setDimension(semanticModel.getDimensionMap()); semanticSchema.setMetric(semanticModel.getMetrics()); semanticSchema.setJoinRelations(semanticModel.getJoinRelations()); - semanticSchema.setRuntimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) - .enableOptimize(queryStatement.getEnableOptimize()).build()); + semanticSchema.setRuntimeOptions( + RuntimeOptions.builder() + .minMaxTime(queryStatement.getMinMaxTime()) + .enableOptimize(queryStatement.getEnableOptimize()) + .build()); return semanticSchema; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java index 8ad84b801..c7997f6b4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java @@ -1,15 +1,10 @@ package com.tencent.supersonic.headless.core.translator.calcite; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlTypeFactoryImpl; import com.tencent.supersonic.headless.core.translator.calcite.schema.ViewExpanderImpl; import com.tencent.supersonic.headless.core.utils.SqlDialectFactory; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; import org.apache.calcite.avatica.util.Casing; import org.apache.calcite.avatica.util.Quoting; import org.apache.calcite.config.CalciteConnectionConfig; @@ -41,15 +36,20 @@ import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; -/** - * global configuration of the calcite - */ +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +/** global configuration of the calcite */ public class Configuration { public static Properties configProperties = new Properties(); - public static RelDataTypeFactory typeFactory = new SemanticSqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + public static RelDataTypeFactory typeFactory = + new SemanticSqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); public static SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); - public static CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties); + public static CalciteConnectionConfig config = + new CalciteConnectionConfigImpl(configProperties); public static SqlValidator.Config getValidatorConfig(EngineType engineType) { SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); @@ -60,10 +60,12 @@ public class Configuration { } static { - configProperties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString()); - configProperties.put(CalciteConnectionProperty.UNQUOTED_CASING.camelName(), Casing.UNCHANGED.toString()); - configProperties.put(CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.TO_LOWER.toString()); - + configProperties.put( + CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString()); + configProperties.put( + CalciteConnectionProperty.UNQUOTED_CASING.camelName(), Casing.UNCHANGED.toString()); + configProperties.put( + CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.TO_LOWER.toString()); } public static SqlParser.Config getParserConfig(EngineType engineType) { @@ -76,7 +78,9 @@ public class Configuration { parserConfig.setQuotedCasing(config.quotedCasing()); parserConfig.setConformance(config.conformance()); parserConfig.setLex(Lex.BIG_QUERY); - parserConfig.setParserFactory(SqlParserImpl.FACTORY).setCaseSensitive(false) + parserConfig + .setParserFactory(SqlParserImpl.FACTORY) + .setCaseSensitive(false) .setIdentifierMaxLength(Integer.MAX_VALUE) .setQuoting(Quoting.BACK_TICK) .setQuoting(Quoting.SINGLE_QUOTE) @@ -93,22 +97,31 @@ public class Configuration { List tables = new ArrayList<>(); tables.add(SqlStdOperatorTable.instance()); SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables); - //operatorTable. - Prepare.CatalogReader catalogReader = new CalciteCatalogReader( - rootSchema, - Collections.singletonList(rootSchema.getName()), + // operatorTable. + Prepare.CatalogReader catalogReader = + new CalciteCatalogReader( + rootSchema, + Collections.singletonList(rootSchema.getName()), + typeFactory, + config); + return SqlValidatorUtil.newValidator( + operatorTable, + catalogReader, typeFactory, - config - ); - return SqlValidatorUtil.newValidator(operatorTable, catalogReader, typeFactory, Configuration.getValidatorConfig(engineType)); } - public static SqlValidatorWithHints getSqlValidatorWithHints(CalciteSchema rootSchema, EngineType engineTyp) { - return new SqlAdvisorValidator(SqlStdOperatorTable.instance(), - new CalciteCatalogReader(rootSchema, - Collections.singletonList(rootSchema.getName()), typeFactory, config), - typeFactory, SqlValidator.Config.DEFAULT); + public static SqlValidatorWithHints getSqlValidatorWithHints( + CalciteSchema rootSchema, EngineType engineTyp) { + return new SqlAdvisorValidator( + SqlStdOperatorTable.instance(), + new CalciteCatalogReader( + rootSchema, + Collections.singletonList(rootSchema.getName()), + typeFactory, + config), + typeFactory, + SqlValidator.Config.DEFAULT); } public static SqlToRelConverter.Config getConverterConfig() { @@ -120,22 +133,29 @@ public class Configuration { .addRelBuilderConfigTransform(c -> c.withSimplify(false)); } - public static SqlToRelConverter getSqlToRelConverter(SqlValidatorScope scope, SqlValidator sqlValidator, - RelOptPlanner relOptPlanner, EngineType engineType) { + public static SqlToRelConverter getSqlToRelConverter( + SqlValidatorScope scope, + SqlValidator sqlValidator, + RelOptPlanner relOptPlanner, + EngineType engineType) { RexBuilder rexBuilder = new RexBuilder(typeFactory); RelOptCluster cluster = RelOptCluster.create(relOptPlanner, rexBuilder); - FrameworkConfig fromworkConfig = Frameworks.newConfigBuilder() - .parserConfig(getParserConfig(engineType)) - .defaultSchema(scope.getValidator().getCatalogReader().getRootSchema().plus()) - .build(); - return new SqlToRelConverter(new ViewExpanderImpl(), + FrameworkConfig fromworkConfig = + Frameworks.newConfigBuilder() + .parserConfig(getParserConfig(engineType)) + .defaultSchema( + scope.getValidator().getCatalogReader().getRootSchema().plus()) + .build(); + return new SqlToRelConverter( + new ViewExpanderImpl(), sqlValidator, - (CatalogReader) scope.getValidator().getCatalogReader(), cluster, fromworkConfig.getConvertletTable(), + (CatalogReader) scope.getValidator().getCatalogReader(), + cluster, + fromworkConfig.getConvertletTable(), getConverterConfig()); } public static SqlAdvisor getSqlAdvisor(SqlValidatorWithHints validator, EngineType engineType) { return new SqlAdvisor(validator, getParserConfig(engineType)); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java index a093abdb1..6497756e2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java @@ -1,23 +1,27 @@ package com.tencent.supersonic.headless.core.translator.calcite.planner; - import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.Configuration; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; +import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; +import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.OutputRender; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.SourceRender; -import com.tencent.supersonic.headless.core.translator.calcite.Configuration; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; -import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; -import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; -import com.tencent.supersonic.headless.core.pojo.Database; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.validate.SqlValidatorScope; + import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -25,14 +29,7 @@ import java.util.ListIterator; import java.util.Objects; import java.util.Stack; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -/** - * parsing from query dimensions and metrics - */ +/** parsing from query dimensions and metrics */ @Slf4j public class AggPlanner implements Planner { @@ -71,15 +68,15 @@ public class AggPlanner implements Planner { Renderer renderer = it.next(); if (previous != null) { previous.render(metricReq, datasource, scope, schema, !isAgg); - renderer.setTable(previous.builderAs(DataSourceNode.getNames(datasource) + "_" + String.valueOf(i))); + renderer.setTable( + previous.builderAs( + DataSourceNode.getNames(datasource) + "_" + String.valueOf(i))); i++; } previous = renderer; } builders.getLast().render(metricReq, datasource, scope, schema, !isAgg); parserNode = builders.getLast().builder(); - - } private List getMatchDataSource(SqlValidatorScope scope) throws Exception { @@ -91,8 +88,10 @@ public class AggPlanner implements Planner { return AggOption.isAgg(aggOption); } // default by dataSource time aggregation - if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime().equalsIgnoreCase( - Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { + if (Objects.nonNull(dataSource.getAggTime()) + && !dataSource + .getAggTime() + .equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { if (!metricReq.isNativeQuery()) { return true; } @@ -137,7 +136,8 @@ public class AggPlanner implements Planner { } public void optimize(EngineType engineType) { - if (Objects.isNull(schema.getRuntimeOptions()) || Objects.isNull(schema.getRuntimeOptions().getEnableOptimize()) + if (Objects.isNull(schema.getRuntimeOptions()) + || Objects.isNull(schema.getRuntimeOptions().getEnableOptimize()) || !schema.getRuntimeOptions().getEnableOptimize()) { return; } @@ -149,9 +149,11 @@ public class AggPlanner implements Planner { public String optimize(String sql, EngineType engineType) { try { - SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); + SqlNode sqlNode = + SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); if (Objects.nonNull(sqlNode)) { - return SemanticNode.getSql(SemanticNode.optimize(scope, schema, sqlNode, engineType), engineType); + return SemanticNode.getSql( + SemanticNode.optimize(scope, schema, sqlNode, engineType), engineType); } } catch (Exception e) { log.error("optimize error {}", e); @@ -161,7 +163,8 @@ public class AggPlanner implements Planner { private SqlNode optimizeSql(String sql, EngineType engineType) { try { - SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); + SqlNode sqlNode = + SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); if (Objects.nonNull(sqlNode)) { return SemanticNode.optimize(scope, schema, sqlNode, engineType); } @@ -170,4 +173,4 @@ public class AggPlanner implements Planner { } return null; } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java index 01fc88a37..36eb0b70e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java @@ -1,13 +1,10 @@ package com.tencent.supersonic.headless.core.translator.calcite.planner; - import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -/** - * parse and generate SQL and other execute information - */ +/** parse and generate SQL and other execute information */ public interface Planner { public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Constants.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Constants.java index 001145806..e2b38952a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Constants.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Constants.java @@ -18,5 +18,4 @@ public class Constants { public static final String SQL_PARSER_DB = "parsed_db"; public static final String SQL_PARSER_FIELD = "parsed_field"; public static final String DIMENSION_DELIMITER = "dimension_delimiter"; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java index 8f4e917ab..300744284 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java @@ -5,7 +5,6 @@ import lombok.Data; import java.util.List; - @Data @Builder public class DataSource { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataType.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataType.java index 36a06eaed..f14e7b737 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataType.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataType.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; import java.util.Arrays; public enum DataType { - ARRAY("ARRAY"), MAP("MAP"), @@ -52,4 +51,4 @@ public enum DataType { public boolean isArray() { return ARRAY.equals(this); } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java index bb565f7b1..98aa5b8f3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java @@ -1,14 +1,11 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; - import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticItem; - -import java.util.List; -import java.util.Map; - import lombok.Builder; import lombok.Data; +import java.util.List; +import java.util.Map; @Data @Builder diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DimensionTimeTypeParams.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DimensionTimeTypeParams.java index 249b1e1e7..396a60528 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DimensionTimeTypeParams.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DimensionTimeTypeParams.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; import lombok.Data; - @Data public class DimensionTimeTypeParams { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Identify.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Identify.java index 277fc3eae..11024c147 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Identify.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Identify.java @@ -4,14 +4,14 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - @Data @AllArgsConstructor @NoArgsConstructor public class Identify { public enum Type { - PRIMARY, FOREIGN + PRIMARY, + FOREIGN } private String name; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/JoinRelation.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/JoinRelation.java index 7eddd0555..80ef7b71c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/JoinRelation.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/JoinRelation.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; -import java.util.List; import lombok.Builder; import lombok.Data; import org.apache.commons.lang3.tuple.Triple; +import java.util.List; + @Data @Builder public class JoinRelation { @@ -13,5 +14,4 @@ public class JoinRelation { private String right; private String joinType; private List> joinCondition; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Materialization.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Materialization.java index a82fa2f89..927c275b6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Materialization.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Materialization.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; +import lombok.Builder; +import lombok.Data; + import java.util.ArrayList; import java.util.List; -import lombok.Builder; -import lombok.Data; @Data @Builder @@ -11,10 +12,8 @@ public class Materialization { public enum TimePartType { /** - * partition time type - * 1 - FULL, not use partition - * 2 - PARTITION , use time list - * 3 - ZIPPER, use [startDate, endDate] range time + * partition time type 1 - FULL, not use partition 2 - PARTITION , use time list 3 - ZIPPER, + * use [startDate, endDate] range time */ FULL("FULL"), PARTITION("PARTITION"), @@ -46,6 +45,4 @@ public class Materialization { private Integer level; private List dimensions = new ArrayList<>(); private List metrics = new ArrayList<>(); - - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MaterializationElement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MaterializationElement.java index 3371f740e..89175e5da 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MaterializationElement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MaterializationElement.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; -import java.util.List; import lombok.Builder; import lombok.Data; +import java.util.List; + @Data @Builder public class MaterializationElement { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Measure.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Measure.java index aad2572ed..20be0f30c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Measure.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Measure.java @@ -5,7 +5,6 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; - @Data @AllArgsConstructor @NoArgsConstructor @@ -14,7 +13,7 @@ public class Measure { private String name; - //sum max min avg count distinct + // sum max min avg count distinct private String agg; private String expr; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java index 114280024..366f2c60c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java @@ -1,11 +1,9 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; - import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticItem; - -import java.util.List; import lombok.Data; +import java.util.List; @Data public class Metric implements SemanticItem { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MetricTypeParams.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MetricTypeParams.java index fa5288f7c..a3386526e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MetricTypeParams.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/MetricTypeParams.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class MetricTypeParams { @@ -11,5 +12,4 @@ public class MetricTypeParams { private List fields; private boolean isFieldMetric = false; private String expr; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java index 6c8bf66ce..71d33ed06 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java @@ -22,11 +22,13 @@ public class SemanticModel { private Database database; public List getDimensions() { - return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + return dimensionMap.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); } public Map getModelMap() { - return datasourceMap.values().stream().collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource)); + return datasourceMap.values().stream() + .collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource)); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java index c4093105b..55bb36ede 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java @@ -1,8 +1,5 @@ package com.tencent.supersonic.headless.core.translator.calcite.schema; - -import java.util.ArrayList; -import java.util.List; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.plan.RelOptTable; @@ -22,9 +19,10 @@ import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.sql.type.SqlTypeName; -/** - * customize the AbstractTable - */ +import java.util.ArrayList; +import java.util.List; + +/** customize the AbstractTable */ public class DataSourceTable extends AbstractTable implements ScannableTable, TranslatableTable { private final String tableName; @@ -34,7 +32,10 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr private RelDataType rowType; - private DataSourceTable(String tableName, List fieldNames, List fieldTypes, + private DataSourceTable( + String tableName, + List fieldNames, + List fieldTypes, Statistic statistic) { this.tableName = tableName; this.fieldNames = fieldNames; @@ -79,10 +80,10 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr public RelNode toRel(RelOptTable.ToRelContext toRelContext, RelOptTable relOptTable) { List hint = new ArrayList<>(); - return new LogicalTableScan(toRelContext.getCluster(), toRelContext.getCluster().traitSet(), hint, relOptTable); + return new LogicalTableScan( + toRelContext.getCluster(), toRelContext.getCluster().traitSet(), hint, relOptTable); } - public static final class Builder { private final String tableName; @@ -127,8 +128,8 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr throw new IllegalStateException("Table must have positive row count"); } - return new DataSourceTable(tableName, fieldNames, fieldTypes, Statistics.of(rowCount, null)); + return new DataSourceTable( + tableName, fieldNames, fieldTypes, Statistics.of(rowCount, null)); } } } - diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java index 09a5833f4..1913f17a3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java @@ -9,5 +9,4 @@ import org.apache.commons.lang3.tuple.Triple; public class RuntimeOptions { private Triple minMaxTime; private Boolean enableOptimize; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java index 01e14df6a..9f7257ecc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.translator.calcite.schema; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.Configuration; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2SQLSqlValidatorImpl; @@ -32,38 +31,50 @@ public class SchemaBuilder { Map nameToTypeMap = new HashMap<>(); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); rootSchema.add(schema.getSchemaKey(), schema); - Prepare.CatalogReader catalogReader = new CalciteCatalogReader( - rootSchema, - Collections.singletonList(schema.getSchemaKey()), - Configuration.typeFactory, - Configuration.config - ); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); - S2SQLSqlValidatorImpl s2SQLSqlValidator = new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader, - Configuration.typeFactory, Configuration.getValidatorConfig(engineType)); + Prepare.CatalogReader catalogReader = + new CalciteCatalogReader( + rootSchema, + Collections.singletonList(schema.getSchemaKey()), + Configuration.typeFactory, + Configuration.config); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + S2SQLSqlValidatorImpl s2SQLSqlValidator = + new S2SQLSqlValidatorImpl( + Configuration.operatorTable, + catalogReader, + Configuration.typeFactory, + Configuration.getValidatorConfig(engineType)); return new ParameterScope(s2SQLSqlValidator, nameToTypeMap); } public static CalciteSchema getMaterializationSchema() { CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); SchemaPlus schema = rootSchema.plus().add(MATERIALIZATION_SYS_DB, new AbstractSchema()); - DataSourceTable srcTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_SOURCE) - .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) - .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT) - .withRowCount(1) - .build(); + DataSourceTable srcTable = + DataSourceTable.newBuilder(MATERIALIZATION_SYS_SOURCE) + .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) + .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT) + .withRowCount(1) + .build(); schema.add(MATERIALIZATION_SYS_SOURCE, srcTable); - DataSourceTable dataSetTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW) - .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) - .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT) - .withRowCount(1) - .build(); + DataSourceTable dataSetTable = + DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW) + .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) + .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT) + .withRowCount(1) + .build(); schema.add(MATERIALIZATION_SYS_VIEW, dataSetTable); return rootSchema; } - public static void addSourceView(CalciteSchema dataSetSchema, String dbSrc, String tbSrc, Set dates, - Set dimensions, Set metrics) { + public static void addSourceView( + CalciteSchema dataSetSchema, + String dbSrc, + String tbSrc, + Set dates, + Set dimensions, + Set metrics) { String tb = tbSrc; String db = dbSrc; DataSourceTable.Builder builder = DataSourceTable.newBuilder(tb); @@ -76,9 +87,7 @@ public class SchemaBuilder { for (String metric : metrics) { builder.addField(metric, SqlTypeName.ANY); } - DataSourceTable srcTable = builder - .withRowCount(1) - .build(); + DataSourceTable srcTable = builder.withRowCount(1).build(); if (Objects.nonNull(db) && !db.isEmpty()) { SchemaPlus dbPs = dataSetSchema.plus(); for (String d : db.split("\\.")) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java index 78ac58ad9..c8afc0cd7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.core.translator.calcite.schema; - -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.SchemaVersion; import org.apache.calcite.schema.Table; @@ -16,7 +15,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - public class SemanticSchema extends AbstractSchema { private final String schemaKey; @@ -28,7 +26,6 @@ public class SemanticSchema extends AbstractSchema { private RuntimeOptions runtimeOptions; - private SemanticSchema(String schemaKey, Map tableMap) { this.schemaKey = schemaKey; this.tableMap = tableMap; @@ -123,7 +120,8 @@ public class SemanticSchema extends AbstractSchema { public Builder addTable(DataSourceTable table) { if (tableMap.containsKey(table.getTableName())) { - throw new IllegalArgumentException("Table already defined: " + table.getTableName()); + throw new IllegalArgumentException( + "Table already defined: " + table.getTableName()); } tableMap.put(table.getTableName(), table); @@ -135,5 +133,4 @@ public class SemanticSchema extends AbstractSchema { return new SemanticSchema(schemaKey, tableMap); } } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlConformance.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlConformance.java index 13c38be9b..3271519dc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlConformance.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlConformance.java @@ -4,9 +4,7 @@ import org.apache.calcite.sql.fun.SqlLibrary; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; -/** - * customize the SqlConformance - */ +/** customize the SqlConformance */ public class SemanticSqlConformance implements SqlConformance { @Override @@ -163,4 +161,4 @@ public class SemanticSqlConformance implements SqlConformance { public boolean allowLenientCoercion() { return false; } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlDialect.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlDialect.java index af087914a..ce83cfdbb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlDialect.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlDialect.java @@ -8,9 +8,7 @@ import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.validate.SqlConformance; import org.checkerframework.checker.nullness.qual.Nullable; -/** - * customize the SqlDialect - */ +/** customize the SqlDialect */ public class SemanticSqlDialect extends SqlDialect { private static final SqlConformance tagTdwSqlConformance = new SemanticSqlConformance(); @@ -19,7 +17,8 @@ public class SemanticSqlDialect extends SqlDialect { super(context); } - public static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { + public static void unparseFetchUsingAnsi( + SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { Preconditions.checkArgument(fetch != null || offset != null); SqlWriter.Frame fetchFrame; writer.newlineAndIndent(); @@ -39,7 +38,6 @@ public class SemanticSqlDialect extends SqlDialect { } writer.endList(fetchFrame); - } @Override @@ -76,10 +74,11 @@ public class SemanticSqlDialect extends SqlDialect { return true; } - public void unparseSqlIntervalLiteral(SqlWriter writer, SqlIntervalLiteral literal, int leftPrec, int rightPrec) { - } + public void unparseSqlIntervalLiteral( + SqlWriter writer, SqlIntervalLiteral literal, int leftPrec, int rightPrec) {} - public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { + public void unparseOffsetFetch( + SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { unparseFetchUsingAnsi(writer, offset, fetch); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlTypeFactoryImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlTypeFactoryImpl.java index ec47e6ed8..f7d36df69 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlTypeFactoryImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlTypeFactoryImpl.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.core.translator.calcite.schema; -import java.nio.charset.Charset; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.sql.type.SqlTypeFactoryImpl; -/** - * customize the SqlTypeFactoryImpl - */ +import java.nio.charset.Charset; + +/** customize the SqlTypeFactoryImpl */ public class SemanticSqlTypeFactoryImpl extends SqlTypeFactoryImpl { public SemanticSqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/ViewExpanderImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/ViewExpanderImpl.java index d122beead..029ce4322 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/ViewExpanderImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/ViewExpanderImpl.java @@ -6,16 +6,16 @@ import org.apache.calcite.rel.type.RelDataType; import java.util.List; -/** - * customize the ViewExpander - */ +/** customize the ViewExpander */ public class ViewExpanderImpl implements RelOptTable.ViewExpander { - public ViewExpanderImpl() { - } + public ViewExpanderImpl() {} @Override - public RelRoot expandView(RelDataType rowType, String queryString, List schemaPath, + public RelRoot expandView( + RelDataType rowType, + String queryString, + List schemaPath, List dataSetPath) { return null; } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java index 6b415b15a..6df9889c2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java @@ -1,9 +1,7 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql; - import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; - public interface Optimization { public void visit(SemanticSchema semanticSchema); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java index cea66ff4f..90cd942e5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; @@ -11,45 +11,56 @@ import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSc import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import lombok.Data; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; + import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.Data; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -/** - * process TableView - */ +/** process TableView */ @Data public abstract class Renderer { protected TableView tableView = new TableView(); public static Optional getDimensionByName(String name, DataSource datasource) { - return datasource.getDimensions().stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); + return datasource.getDimensions().stream() + .filter(d -> d.getName().equalsIgnoreCase(name)) + .findFirst(); } public static Optional getMeasureByName(String name, DataSource datasource) { - return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name)).findFirst(); + return datasource.getMeasures().stream() + .filter(mm -> mm.getName().equalsIgnoreCase(name)) + .findFirst(); } public static Optional getMetricByName(String name, SemanticSchema schema) { - Optional metric = schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(name)) - .findFirst(); + Optional metric = + schema.getMetrics().stream() + .filter(m -> m.getName().equalsIgnoreCase(name)) + .findFirst(); return metric; } public static Optional getIdentifyByName(String name, DataSource datasource) { - return datasource.getIdentifiers().stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); + return datasource.getIdentifiers().stream() + .filter(i -> i.getName().equalsIgnoreCase(name)) + .findFirst(); } - public static MetricNode buildMetricNode(String metric, DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg, String alias) throws Exception { + public static MetricNode buildMetricNode( + String metric, + DataSource datasource, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg, + String alias) + throws Exception { Optional metricOpt = getMetricByName(metric, schema); MetricNode metricNode = new MetricNode(); EngineType engineType = EngineType.fromString(datasource.getType()); @@ -58,37 +69,61 @@ public abstract class Renderer { for (Measure m : metricOpt.get().getMetricTypeParams().getMeasures()) { Optional measure = getMeasureByName(m.getName(), datasource); if (measure.isPresent()) { - metricNode.getNonAggNode() - .put(measure.get().getName(), - MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType)); - metricNode.getAggNode() - .put(measure.get().getName(), + metricNode + .getNonAggNode() + .put( + measure.get().getName(), + MeasureNode.buildNonAgg( + alias, measure.get(), scope, engineType)); + metricNode + .getAggNode() + .put( + measure.get().getName(), MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType)); - metricNode.getAggFunction().put(measure.get().getName(), measure.get().getAgg()); + metricNode + .getAggFunction() + .put(measure.get().getName(), measure.get().getAgg()); } else { - metricNode.getNonAggNode().put(m.getName(), MeasureNode.buildNonAgg(alias, m, scope, engineType)); - metricNode.getAggNode().put(m.getName(), MeasureNode.buildAgg(m, nonAgg, scope, engineType)); + metricNode + .getNonAggNode() + .put(m.getName(), MeasureNode.buildNonAgg(alias, m, scope, engineType)); + metricNode + .getAggNode() + .put(m.getName(), MeasureNode.buildAgg(m, nonAgg, scope, engineType)); metricNode.getAggFunction().put(m.getName(), m.getAgg()); } if (m.getConstraint() != null && !m.getConstraint().isEmpty()) { - metricNode.getMeasureFilter() - .put(m.getName(), SemanticNode.parse(m.getConstraint(), scope, engineType)); + metricNode + .getMeasureFilter() + .put( + m.getName(), + SemanticNode.parse(m.getConstraint(), scope, engineType)); } } return metricNode; } Optional measure = getMeasureByName(metric, datasource); if (measure.isPresent()) { - metricNode.getNonAggNode() - .put(measure.get().getName(), MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType)); - metricNode.getAggNode() - .put(measure.get().getName(), MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType)); + metricNode + .getNonAggNode() + .put( + measure.get().getName(), + MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType)); + metricNode + .getAggNode() + .put( + measure.get().getName(), + MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType)); metricNode.getAggFunction().put(measure.get().getName(), measure.get().getAgg()); if (measure.get().getConstraint() != null && !measure.get().getConstraint().isEmpty()) { - metricNode.getMeasureFilter().put(measure.get().getName(), - SemanticNode.parse(measure.get().getConstraint(), scope, engineType)); + metricNode + .getMeasureFilter() + .put( + measure.get().getName(), + SemanticNode.parse( + measure.get().getConstraint(), scope, engineType)); } } return metricNode; @@ -111,6 +146,11 @@ public abstract class Renderer { return SemanticNode.buildAs(alias, tableView.build()); } - public abstract void render(MetricQueryParam metricCommand, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception; + public abstract void render( + MetricQueryParam metricCommand, + List dataSources, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2SQLSqlValidatorImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2SQLSqlValidatorImpl.java index 985120b1c..a3d5c3eda 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2SQLSqlValidatorImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2SQLSqlValidatorImpl.java @@ -5,13 +5,14 @@ import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; import org.apache.calcite.sql.validate.SqlValidatorImpl; -/** - * customize the SqlValidatorImpl - */ +/** customize the SqlValidatorImpl */ public class S2SQLSqlValidatorImpl extends SqlValidatorImpl { - public S2SQLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader, - RelDataTypeFactory typeFactory, Config config) { + public S2SQLSqlValidatorImpl( + SqlOperatorTable opTab, + SqlValidatorCatalogReader catalogReader, + RelDataTypeFactory typeFactory, + Config config) { super(opTab, catalogReader, typeFactory, config); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java index f1d509f34..fd784a88c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java @@ -1,9 +1,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; import lombok.Data; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlKind; @@ -12,9 +9,11 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParserPos; -/** - * basic query project - */ +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** basic query project */ @Data public class TableView { @@ -40,15 +39,29 @@ public class TableView { if (filter.size() > 0) { filterNodeList = new SqlNodeList(filter, SqlParserPos.ZERO); } - return new SqlSelect(SqlParserPos.ZERO, null, new SqlNodeList(measure, SqlParserPos.ZERO), table, - filterNodeList, dimensionNodeList, null, null, null, order, offset, fetch, null); + return new SqlSelect( + SqlParserPos.ZERO, + null, + new SqlNodeList(measure, SqlParserPos.ZERO), + table, + filterNodeList, + dimensionNodeList, + null, + null, + null, + order, + offset, + fetch, + null); } private List getGroup(List sqlNodeList) { return sqlNodeList.stream() - .map(s -> (s.getKind().equals(SqlKind.AS) ? ((SqlBasicCall) s).getOperandList().get(0) : s)) - .collect( - Collectors.toList()); + .map( + s -> + (s.getKind().equals(SqlKind.AS) + ? ((SqlBasicCall) s).getOperandList().get(0) + : s)) + .collect(Collectors.toList()); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/AggFunctionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/AggFunctionNode.java index ff6b1639e..68520c923 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/AggFunctionNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/AggFunctionNode.java @@ -1,19 +1,28 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import java.util.Objects; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; +import java.util.Objects; + public class AggFunctionNode extends SemanticNode { - public static SqlNode build(String agg, String name, SqlValidatorScope scope, EngineType engineType) + public static SqlNode build( + String agg, String name, SqlValidatorScope scope, EngineType engineType) throws Exception { if (Objects.isNull(agg) || agg.isEmpty()) { return parse(name, scope, engineType); } if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) { - return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name + " ) ", scope, + return parse( + AggFunction.COUNT.name() + + " ( " + + AggFunction.DISTINCT.name() + + " " + + name + + " ) ", + scope, engineType); } return parse(agg + " ( " + name + " ) ", scope, engineType); @@ -28,6 +37,4 @@ public class AggFunctionNode extends SemanticNode { COUNT, DISTINCT } - - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java index 081e7285f..c3c60c464 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java @@ -3,7 +3,10 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.Configuration; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; @@ -11,9 +14,17 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.extend.LateralViewExplodeNode; -import com.tencent.supersonic.headless.core.translator.calcite.Configuration; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -26,17 +37,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.springframework.util.CollectionUtils; - @Slf4j public class DataSourceNode extends SemanticNode { @@ -46,8 +46,9 @@ public class DataSourceNode extends SemanticNode { sqlTable = datasource.getSqlQuery(); } else if (datasource.getTableQuery() != null && !datasource.getTableQuery().isEmpty()) { if (datasource.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { - String fullTableName = Arrays.stream(datasource.getTableQuery().split("\\.")) - .collect(Collectors.joining(".public.")); + String fullTableName = + Arrays.stream(datasource.getTableQuery().split("\\.")) + .collect(Collectors.joining(".public.")); sqlTable = "select * from " + fullTableName; } else { sqlTable = "select * from " + datasource.getTableQuery(); @@ -61,7 +62,8 @@ public class DataSourceNode extends SemanticNode { return buildAs(datasource.getName(), source); } - private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) throws Exception { + private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) + throws Exception { Map> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table); for (Map.Entry> entry : sqlTable.entrySet()) { String tb = entry.getKey(); @@ -74,7 +76,11 @@ public class DataSourceNode extends SemanticNode { } } - private static void addSchemaTable(SqlValidatorScope scope, DataSource datasource, String db, String tb, + private static void addSchemaTable( + SqlValidatorScope scope, + DataSource datasource, + String db, + String tb, Set fields) throws Exception { Set dateInfo = new HashSet<>(); @@ -82,7 +88,8 @@ public class DataSourceNode extends SemanticNode { Set metrics = new HashSet<>(); EngineType engineType = EngineType.fromString(datasource.getType()); for (Dimension d : datasource.getDimensions()) { - List identifiers = expand(SemanticNode.parse(d.getExpr(), scope, engineType), scope); + List identifiers = + expand(SemanticNode.parse(d.getExpr(), scope, engineType), scope); identifiers.stream().forEach(i -> dimensions.add(i.toString())); dimensions.add(d.getName()); } @@ -90,13 +97,15 @@ public class DataSourceNode extends SemanticNode { dimensions.add(i.getName()); } for (Measure m : datasource.getMeasures()) { - List identifiers = expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope); - identifiers.stream().forEach(i -> { - if (!dimensions.contains(i.toString())) { - metrics.add(i.toString()); - } - } - ); + List identifiers = + expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope); + identifiers.stream() + .forEach( + i -> { + if (!dimensions.contains(i.toString())) { + metrics.add(i.toString()); + } + }); if (!dimensions.contains(m.getName())) { metrics.add(m.getName()); } @@ -107,37 +116,52 @@ public class DataSourceNode extends SemanticNode { log.info("add column {} {}", datasource.getName(), field); } } - SchemaBuilder.addSourceView(scope.getValidator().getCatalogReader().getRootSchema(), db, - tb, dateInfo, - dimensions, metrics); + SchemaBuilder.addSourceView( + scope.getValidator().getCatalogReader().getRootSchema(), + db, + tb, + dateInfo, + dimensions, + metrics); } - public static SqlNode buildExtend(DataSource datasource, Map exprList, - SqlValidatorScope scope) + public static SqlNode buildExtend( + DataSource datasource, Map exprList, SqlValidatorScope scope) throws Exception { if (CollectionUtils.isEmpty(exprList)) { return build(datasource, scope); } EngineType engineType = EngineType.fromString(datasource.getType()); - SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(exprList), Arrays.asList(build(datasource, scope), - new SqlNodeList(getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)), SqlParserPos.ZERO); + SqlNode dataSet = + new SqlBasicCall( + new LateralViewExplodeNode(exprList), + Arrays.asList( + build(datasource, scope), + new SqlNodeList( + getExtendField(exprList, scope, engineType), + SqlParserPos.ZERO)), + SqlParserPos.ZERO); return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, dataSet); } - public static List getExtendField(Map exprList, SqlValidatorScope scope, - EngineType engineType) + public static List getExtendField( + Map exprList, SqlValidatorScope scope, EngineType engineType) throws Exception { List sqlNodeList = new ArrayList<>(); for (String expr : exprList.keySet()) { sqlNodeList.add(parse(expr, scope, engineType)); - sqlNodeList.add(new SqlDataTypeSpec( - new SqlUserDefinedTypeNameSpec(expr + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, SqlParserPos.ZERO), - SqlParserPos.ZERO)); + sqlNodeList.add( + new SqlDataTypeSpec( + new SqlUserDefinedTypeNameSpec( + expr + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, + SqlParserPos.ZERO), + SqlParserPos.ZERO)); } return sqlNodeList; } - private static SqlNode getTable(String sqlQuery, SqlValidatorScope scope, EngineType engineType) throws Exception { + private static SqlNode getTable(String sqlQuery, SqlValidatorScope scope, EngineType engineType) + throws Exception { SqlParser sqlParser = SqlParser.create(sqlQuery, Configuration.getParserConfig(engineType)); SqlNode sqlNode = sqlParser.parseQuery(); scope.validateExpr(sqlNode); @@ -148,33 +172,57 @@ public class DataSourceNode extends SemanticNode { return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); } - public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryParam metricCommand, - Set queryDimension, List measures) { - queryDimension.addAll(metricCommand.getDimensions().stream() - .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d) - .collect(Collectors.toSet())); - Set schemaMetricName = schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet()); - schema.getMetrics().stream().filter(m -> metricCommand.getMetrics().contains(m.getName())) - .forEach(m -> m.getMetricTypeParams().getMeasures().stream().forEach(mm -> measures.add(mm.getName()))); - metricCommand.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)).forEach(m -> measures.add(m)); - + public static void getQueryDimensionMeasure( + SemanticSchema schema, + MetricQueryParam metricCommand, + Set queryDimension, + List measures) { + queryDimension.addAll( + metricCommand.getDimensions().stream() + .map( + d -> + d.contains(Constants.DIMENSION_IDENTIFY) + ? d.split(Constants.DIMENSION_IDENTIFY)[1] + : d) + .collect(Collectors.toSet())); + Set schemaMetricName = + schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet()); + schema.getMetrics().stream() + .filter(m -> metricCommand.getMetrics().contains(m.getName())) + .forEach( + m -> + m.getMetricTypeParams().getMeasures().stream() + .forEach(mm -> measures.add(mm.getName()))); + metricCommand.getMetrics().stream() + .filter(m -> !schemaMetricName.contains(m)) + .forEach(m -> measures.add(m)); } - public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryParam metricCommand, - Set queryDimension, List measures, - SqlValidatorScope scope) throws Exception { - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + public static void mergeQueryFilterDimensionMeasure( + SemanticSchema schema, + MetricQueryParam metricCommand, + Set queryDimension, + List measures, + SqlValidatorScope scope) + throws Exception { + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { Set filterConditions = new HashSet<>(); - FilterNode.getFilterField(parse(metricCommand.getWhere(), scope, engineType), filterConditions); + FilterNode.getFilterField( + parse(metricCommand.getWhere(), scope, engineType), filterConditions); Set queryMeasures = new HashSet<>(measures); - Set schemaMetricName = schema.getMetrics().stream() - .map(m -> m.getName()).collect(Collectors.toSet()); + Set schemaMetricName = + schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet()); for (String filterCondition : filterConditions) { if (schemaMetricName.contains(filterCondition)) { - schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(filterCondition)) - .forEach(m -> m.getMetricTypeParams().getMeasures().stream() - .forEach(mm -> queryMeasures.add(mm.getName()))); + schema.getMetrics().stream() + .filter(m -> m.getName().equalsIgnoreCase(filterCondition)) + .forEach( + m -> + m.getMetricTypeParams().getMeasures().stream() + .forEach( + mm -> queryMeasures.add(mm.getName()))); continue; } queryDimension.add(filterCondition); @@ -184,8 +232,9 @@ public class DataSourceNode extends SemanticNode { } } - public static List getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema, - MetricQueryParam metricCommand) throws Exception { + public static List getMatchDataSources( + SqlValidatorScope scope, SemanticSchema schema, MetricQueryParam metricCommand) + throws Exception { List dataSources = new ArrayList<>(); // check by metric @@ -196,14 +245,18 @@ public class DataSourceNode extends SemanticNode { // one , match measure count Map dataSourceMeasures = new HashMap<>(); for (Map.Entry entry : schema.getDatasource().entrySet()) { - Set sourceMeasure = entry.getValue().getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); + Set sourceMeasure = + entry.getValue().getMeasures().stream() + .map(mm -> mm.getName()) + .collect(Collectors.toSet()); sourceMeasure.retainAll(measures); dataSourceMeasures.put(entry.getKey(), sourceMeasure.size()); } log.info("dataSourceMeasures [{}]", dataSourceMeasures); - Optional> base = dataSourceMeasures.entrySet().stream() - .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst(); + Optional> base = + dataSourceMeasures.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) + .findFirst(); if (base.isPresent()) { baseDataSource = schema.getDatasource().get(base.get().getKey()); dataSources.add(baseDataSource); @@ -211,57 +264,79 @@ public class DataSourceNode extends SemanticNode { // second , check match all dimension and metric if (baseDataSource != null) { Set filterMeasure = new HashSet<>(); - Set sourceMeasure = baseDataSource.getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); - Set dimension = baseDataSource.getDimensions().stream().map(dd -> dd.getName()) - .collect(Collectors.toSet()); + Set sourceMeasure = + baseDataSource.getMeasures().stream() + .map(mm -> mm.getName()) + .collect(Collectors.toSet()); + Set dimension = + baseDataSource.getDimensions().stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); baseDataSource.getIdentifiers().stream().forEach(i -> dimension.add(i.getName())); if (schema.getDimension().containsKey(baseDataSource.getName())) { - schema.getDimension().get(baseDataSource.getName()).stream().forEach(d -> dimension.add(d.getName())); + schema.getDimension().get(baseDataSource.getName()).stream() + .forEach(d -> dimension.add(d.getName())); } filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(dimension); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); - mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures, scope); - boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, metricCommand, scope, - engineType); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + mergeQueryFilterDimensionMeasure( + schema, metricCommand, queryDimension, measures, scope); + boolean isAllMatch = + checkMatch( + sourceMeasure, + queryDimension, + measures, + dimension, + metricCommand, + scope, + engineType); if (isAllMatch) { log.debug("baseDataSource match all "); return dataSources; } // find all dataSource has the same identifiers - List linkDataSources = getLinkDataSourcesByJoinRelation(queryDimension, measures, - baseDataSource, schema); + List linkDataSources = + getLinkDataSourcesByJoinRelation( + queryDimension, measures, baseDataSource, schema); if (CollectionUtils.isEmpty(linkDataSources)) { log.debug("baseDataSource get by identifiers "); - Set baseIdentifiers = baseDataSource.getIdentifiers().stream().map(i -> i.getName()) - .collect(Collectors.toSet()); + Set baseIdentifiers = + baseDataSource.getIdentifiers().stream() + .map(i -> i.getName()) + .collect(Collectors.toSet()); if (baseIdentifiers.isEmpty()) { - throw new Exception("datasource error : " + baseDataSource.getName() + " miss identifier"); + throw new Exception( + "datasource error : " + baseDataSource.getName() + " miss identifier"); } - linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures, - baseDataSource, schema); + linkDataSources = + getLinkDataSources( + baseIdentifiers, queryDimension, measures, baseDataSource, schema); if (linkDataSources.isEmpty()) { throw new Exception( - String.format("not find the match datasource : dimension[%s],measure[%s]", queryDimension, - measures)); + String.format( + "not find the match datasource : dimension[%s],measure[%s]", + queryDimension, measures)); } } log.debug("linkDataSources {}", linkDataSources); return linkDataSources; - //dataSources.addAll(linkDataSources); + // dataSources.addAll(linkDataSources); } return dataSources; } - private static boolean checkMatch(Set sourceMeasure, + private static boolean checkMatch( + Set sourceMeasure, Set queryDimension, List measures, Set dimension, MetricQueryParam metricCommand, SqlValidatorScope scope, - EngineType engineType) throws Exception { + EngineType engineType) + throws Exception { boolean isAllMatch = true; sourceMeasure.retainAll(measures); if (sourceMeasure.size() < measures.size()) { @@ -292,8 +367,11 @@ public class DataSourceNode extends SemanticNode { return isAllMatch; } - private static List getLinkDataSourcesByJoinRelation(Set queryDimension, List measures, - DataSource baseDataSource, SemanticSchema schema) { + private static List getLinkDataSourcesByJoinRelation( + Set queryDimension, + List measures, + DataSource baseDataSource, + SemanticSchema schema) { Set linkDataSourceName = new HashSet<>(); List linkDataSources = new ArrayList<>(); Set before = new HashSet<>(); @@ -301,37 +379,49 @@ public class DataSourceNode extends SemanticNode { if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { Set visitJoinRelations = new HashSet<>(); List sortedJoinRelation = new ArrayList<>(); - sortJoinRelation(schema.getJoinRelations(), baseDataSource.getName(), visitJoinRelations, + sortJoinRelation( + schema.getJoinRelations(), + baseDataSource.getName(), + visitJoinRelations, sortedJoinRelation); - schema.getJoinRelations().stream().filter(j -> !visitJoinRelations.contains(j.getId())) + schema.getJoinRelations().stream() + .filter(j -> !visitJoinRelations.contains(j.getId())) .forEach(j -> sortedJoinRelation.add(j)); for (JoinRelation joinRelation : sortedJoinRelation) { - if (!before.contains(joinRelation.getLeft()) && !before.contains(joinRelation.getRight())) { + if (!before.contains(joinRelation.getLeft()) + && !before.contains(joinRelation.getRight())) { continue; } boolean isMatch = false; boolean isRight = before.contains(joinRelation.getLeft()); - DataSource other = isRight ? schema.getDatasource().get(joinRelation.getRight()) - : schema.getDatasource().get(joinRelation.getLeft()); + DataSource other = + isRight + ? schema.getDatasource().get(joinRelation.getRight()) + : schema.getDatasource().get(joinRelation.getLeft()); if (!queryDimension.isEmpty()) { - Set linkDimension = other.getDimensions().stream().map(dd -> dd.getName()) - .collect(Collectors.toSet()); + Set linkDimension = + other.getDimensions().stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); other.getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName())); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { isMatch = true; } } - Set linkMeasure = other.getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); + Set linkMeasure = + other.getMeasures().stream() + .map(mm -> mm.getName()) + .collect(Collectors.toSet()); linkMeasure.retainAll(measures); if (!linkMeasure.isEmpty()) { isMatch = true; } if (!isMatch && schema.getDimension().containsKey(other.getName())) { - Set linkDimension = schema.getDimension().get(other.getName()).stream() - .map(dd -> dd.getName()) - .collect(Collectors.toSet()); + Set linkDimension = + schema.getDimension().get(other.getName()).stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { isMatch = true; @@ -348,27 +438,35 @@ public class DataSourceNode extends SemanticNode { linkDataSourceName.add(baseDataSource.getName()); orders.put(baseDataSource.getName(), 0L); for (JoinRelation joinRelation : schema.getJoinRelations()) { - if (linkDataSourceName.contains(joinRelation.getLeft()) && linkDataSourceName.contains( - joinRelation.getRight())) { + if (linkDataSourceName.contains(joinRelation.getLeft()) + && linkDataSourceName.contains(joinRelation.getRight())) { orders.put(joinRelation.getLeft(), 0L); orders.put(joinRelation.getRight(), 1L); } } - orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> { - linkDataSources.add(schema.getDatasource().get(d.getKey())); - }); + orders.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .forEach( + d -> { + linkDataSources.add(schema.getDatasource().get(d.getKey())); + }); } return linkDataSources; } - private static void sortJoinRelation(List joinRelations, String next, Set visited, + private static void sortJoinRelation( + List joinRelations, + String next, + Set visited, List sortedJoins) { for (JoinRelation link : joinRelations) { if (!visited.contains(link.getId())) { if (link.getLeft().equals(next) || link.getRight().equals(next)) { visited.add(link.getId()); sortedJoins.add(link); - sortJoinRelation(joinRelations, link.getLeft().equals(next) ? link.getRight() : link.getLeft(), + sortJoinRelation( + joinRelations, + link.getLeft().equals(next) ? link.getRight() : link.getLeft(), visited, sortedJoins); } @@ -376,7 +474,8 @@ public class DataSourceNode extends SemanticNode { } } - private static List getLinkDataSources(Set baseIdentifiers, + private static List getLinkDataSources( + Set baseIdentifiers, Set queryDimension, List measures, DataSource baseDataSource, @@ -387,22 +486,30 @@ public class DataSourceNode extends SemanticNode { if (entry.getKey().equalsIgnoreCase(baseDataSource.getName())) { continue; } - Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName()) - .filter(i -> baseIdentifiers.contains(i)).count(); + Long identifierNum = + entry.getValue().getIdentifiers().stream() + .map(i -> i.getName()) + .filter(i -> baseIdentifiers.contains(i)) + .count(); if (identifierNum > 0) { boolean isMatch = false; if (!queryDimension.isEmpty()) { - Set linkDimension = entry.getValue().getDimensions().stream().map(dd -> dd.getName()) - .collect(Collectors.toSet()); - entry.getValue().getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName())); + Set linkDimension = + entry.getValue().getDimensions().stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); + entry.getValue().getIdentifiers().stream() + .forEach(i -> linkDimension.add(i.getName())); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { isMatch = true; } } if (!measures.isEmpty()) { - Set linkMeasure = entry.getValue().getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); + Set linkMeasure = + entry.getValue().getMeasures().stream() + .map(mm -> mm.getName()) + .collect(Collectors.toSet()); linkMeasure.retainAll(measures); if (!linkMeasure.isEmpty()) { isMatch = true; @@ -415,8 +522,10 @@ public class DataSourceNode extends SemanticNode { } for (Map.Entry> entry : schema.getDimension().entrySet()) { if (!queryDimension.isEmpty()) { - Set linkDimension = entry.getValue().stream().map(dd -> dd.getName()) - .collect(Collectors.toSet()); + Set linkDimension = + entry.getValue().stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { linkDataSourceName.add(entry.getKey()); @@ -434,4 +543,4 @@ public class DataSourceNode extends SemanticNode { } return Lists.newArrayList(); } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DimensionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DimensionNode.java index a7df9ba42..4106a4faa 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DimensionNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DimensionNode.java @@ -1,40 +1,40 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; import java.util.List; import java.util.Objects; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - public class DimensionNode extends SemanticNode { - public static SqlNode build(Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { + public static SqlNode build(Dimension dimension, SqlValidatorScope scope, EngineType engineType) + throws Exception { SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); return buildAs(dimension.getName(), sqlNode); } - public static List expand(Dimension dimension, SqlValidatorScope scope, EngineType engineType) - throws Exception { + public static List expand( + Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); return expand(sqlNode, scope); } - public static SqlNode buildName(Dimension dimension, SqlValidatorScope scope, EngineType engineType) - throws Exception { + public static SqlNode buildName( + Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { return parse(dimension.getName(), scope, engineType); } - public static SqlNode buildExp(Dimension dimension, SqlValidatorScope scope, EngineType engineType) - throws Exception { + public static SqlNode buildExp( + Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { return parse(dimension.getExpr(), scope, engineType); } - public static SqlNode buildNameAs(String alias, Dimension dimension, SqlValidatorScope scope, EngineType engineType) + public static SqlNode buildNameAs( + String alias, Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { if ("".equals(alias)) { return buildName(dimension, scope, engineType); @@ -43,17 +43,20 @@ public class DimensionNode extends SemanticNode { return buildAs(alias, sqlNode); } - public static SqlNode buildArray(Dimension dimension, SqlValidatorScope scope, EngineType engineType) - throws Exception { + public static SqlNode buildArray( + Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { if (Objects.nonNull(dimension.getDataType()) && dimension.getDataType().isArray()) { SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); if (isIdentifier(sqlNode)) { - return buildAs(dimension.getName(), - parse(dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, scope, engineType)); + return buildAs( + dimension.getName(), + parse( + dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, + scope, + engineType)); } throw new Exception("array dimension expr should only identify"); } return build(dimension, scope, engineType); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/ExtendNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/ExtendNode.java index 70b83b47f..d0a6b6e47 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/ExtendNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/ExtendNode.java @@ -32,4 +32,4 @@ public class ExtendNode extends SqlInternalOperator { writer.endList(frameArgs); writer.endList(frame); } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/FilterNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/FilterNode.java index d84b78827..ba8993de1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/FilterNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/FilterNode.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; -import java.util.Set; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlNode; +import java.util.Set; + public class FilterNode extends SemanticNode { public static void getFilterField(SqlNode sqlNode, Set fields) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/IdentifyNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/IdentifyNode.java index b100b525f..a76b362ec 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/IdentifyNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/IdentifyNode.java @@ -2,30 +2,31 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - public class IdentifyNode extends SemanticNode { - public static SqlNode build(Identify identify, SqlValidatorScope scope, EngineType engineType) throws Exception { + public static SqlNode build(Identify identify, SqlValidatorScope scope, EngineType engineType) + throws Exception { return parse(identify.getName(), scope, engineType); } public static Set getIdentifyNames(List identifies, Identify.Type type) { - return identifies.stream().filter(i -> type.name().equalsIgnoreCase(i.getType())).map(i -> i.getName()) + return identifies.stream() + .filter(i -> type.name().equalsIgnoreCase(i.getType())) + .map(i -> i.getName()) .collect(Collectors.toSet()); - } public static boolean isForeign(String name, List identifies) { - Optional identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)) - .findFirst(); + Optional identify = + identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); if (identify.isPresent()) { return Identify.Type.FOREIGN.name().equalsIgnoreCase(identify.get().getType()); } @@ -33,8 +34,8 @@ public class IdentifyNode extends SemanticNode { } public static boolean isPrimary(String name, List identifies) { - Optional identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)) - .findFirst(); + Optional identify = + identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); if (identify.isPresent()) { return Identify.Type.PRIMARY.name().equalsIgnoreCase(identify.get().getType()); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MeasureNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MeasureNode.java index 356a0fb5b..90beccda1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MeasureNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MeasureNode.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import org.apache.calcite.sql.SqlNode; @@ -8,26 +7,30 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; public class MeasureNode extends SemanticNode { - public static SqlNode buildNonAgg(String alias, Measure measure, SqlValidatorScope scope, EngineType engineType) + public static SqlNode buildNonAgg( + String alias, Measure measure, SqlValidatorScope scope, EngineType engineType) throws Exception { return buildAs(measure.getName(), getExpr(measure, alias, scope, engineType)); } - public static SqlNode buildAgg(Measure measure, boolean noAgg, SqlValidatorScope scope, EngineType engineType) + public static SqlNode buildAgg( + Measure measure, boolean noAgg, SqlValidatorScope scope, EngineType engineType) throws Exception { if ((measure.getAgg() == null || measure.getAgg().isEmpty()) || noAgg) { return parse(measure.getName(), scope, engineType); } - return buildAs(measure.getName(), + return buildAs( + measure.getName(), AggFunctionNode.build(measure.getAgg(), measure.getName(), scope, engineType)); } - private static SqlNode getExpr(Measure measure, String alias, SqlValidatorScope scope, EngineType enginType) + private static SqlNode getExpr( + Measure measure, String alias, SqlValidatorScope scope, EngineType enginType) throws Exception { if (measure.getExpr() == null) { - return parse((alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope, enginType); + return parse( + (alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope, enginType); } return parse(measure.getExpr(), scope, enginType); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java index dbd9133d8..97bf3040a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java @@ -1,18 +1,16 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import lombok.Data; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import lombok.Data; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - @Data public class MetricNode extends SemanticNode { @@ -22,8 +20,10 @@ public class MetricNode extends SemanticNode { private Map measureFilter = new HashMap<>(); private Map aggFunction = new HashMap<>(); - public static SqlNode build(Metric metric, SqlValidatorScope scope, EngineType engineType) throws Exception { - if (metric.getMetricTypeParams() == null || metric.getMetricTypeParams().getExpr() == null + public static SqlNode build(Metric metric, SqlValidatorScope scope, EngineType engineType) + throws Exception { + if (metric.getMetricTypeParams() == null + || metric.getMetricTypeParams().getExpr() == null || metric.getMetricTypeParams().getExpr().isEmpty()) { return parse(metric.getName(), scope, engineType); } @@ -32,13 +32,14 @@ public class MetricNode extends SemanticNode { } public static Boolean isMetricField(String name, SemanticSchema schema) { - Optional metric = schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(name)) - .findFirst(); + Optional metric = + schema.getMetrics().stream() + .filter(m -> m.getName().equalsIgnoreCase(name)) + .findFirst(); return metric.isPresent() && metric.get().getMetricTypeParams().isFieldMetric(); } public static Boolean isMetricField(Metric metric) { return metric.getMetricTypeParams().isFieldMetric(); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java index 802ad358c..15303d23b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.core.translator.calcite.Configuration; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer.FilterToGroupScanRule; -import com.tencent.supersonic.headless.core.translator.calcite.Configuration; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlDialect; +import com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer.FilterToGroupScanRule; import com.tencent.supersonic.headless.core.utils.SqlDialectFactory; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.plan.RelOptPlanner; @@ -52,15 +51,14 @@ import java.util.function.UnaryOperator; import java.util.regex.Pattern; import java.util.stream.Collectors; -/** - * model item node - */ +/** model item node */ @Slf4j public abstract class SemanticNode { public static Set AGGREGATION_KIND = new HashSet<>(); public static Set AGGREGATION_FUNC = new HashSet<>(); - public static List groupHints = new ArrayList<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9")); + public static List groupHints = + new ArrayList<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9")); static { AGGREGATION_KIND.add(SqlKind.AVG); @@ -76,14 +74,18 @@ public abstract class SemanticNode { AGGREGATION_FUNC.add("min"); } - public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType) throws Exception { - SqlValidatorWithHints sqlValidatorWithHints = Configuration.getSqlValidatorWithHints( - scope.getValidator().getCatalogReader().getRootSchema(), engineType); - if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords() + public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType) + throws Exception { + SqlValidatorWithHints sqlValidatorWithHints = + Configuration.getSqlValidatorWithHints( + scope.getValidator().getCatalogReader().getRootSchema(), engineType); + if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType) + .getReservedAndKeyWords() .contains(expression.toUpperCase())) { expression = String.format("`%s`", expression); } - SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig(engineType)); + SqlParser sqlParser = + SqlParser.create(expression, Configuration.getParserConfig(engineType)); SqlNode sqlNode = sqlParser.parseExpression(); scope.validateExpr(sqlNode); return sqlNode; @@ -92,12 +94,15 @@ public abstract class SemanticNode { public static SqlNode buildAs(String asName, SqlNode sqlNode) throws Exception { SqlAsOperator sqlAsOperator = new SqlAsOperator(); SqlIdentifier sqlIdentifier = new SqlIdentifier(asName, SqlParserPos.ZERO); - return new SqlBasicCall(sqlAsOperator, new ArrayList<>(Arrays.asList(sqlNode, sqlIdentifier)), + return new SqlBasicCall( + sqlAsOperator, + new ArrayList<>(Arrays.asList(sqlNode, sqlIdentifier)), SqlParserPos.ZERO); } public static String getSql(SqlNode sqlNode, EngineType engineType) { - UnaryOperator sqlWriterConfigUnaryOperator = (c) -> getSqlWriterConfig(engineType); + UnaryOperator sqlWriterConfigUnaryOperator = + (c) -> getSqlWriterConfig(engineType); return sqlNode.toSqlString(sqlWriterConfigUnaryOperator).getSql(); } @@ -134,7 +139,8 @@ public abstract class SemanticNode { public static SqlNode getAlias(SqlNode sqlNode, SqlValidatorScope scope) throws Exception { if (sqlNode instanceof SqlBasicCall) { SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode; - if (sqlBasicCall.getKind().equals(SqlKind.AS) && sqlBasicCall.getOperandList().size() > 1) { + if (sqlBasicCall.getKind().equals(SqlKind.AS) + && sqlBasicCall.getOperandList().size() > 1) { return sqlBasicCall.getOperandList().get(1); } } @@ -149,9 +155,17 @@ public abstract class SemanticNode { if (table instanceof SqlSelect) { SqlSelect tableSelect = (SqlSelect) table; return tableSelect.getSelectList().stream() - .map(s -> (s instanceof SqlIdentifier) ? ((SqlIdentifier) s).names.get(0) - : (((s instanceof SqlBasicCall) && s.getKind().equals(SqlKind.AS)) - ? ((SqlBasicCall) s).getOperandList().get(1).toString() : "")) + .map( + s -> + (s instanceof SqlIdentifier) + ? ((SqlIdentifier) s).names.get(0) + : (((s instanceof SqlBasicCall) + && s.getKind().equals(SqlKind.AS)) + ? ((SqlBasicCall) s) + .getOperandList() + .get(1) + .toString() + : "")) .collect(Collectors.toSet()); } return new HashSet<>(); @@ -172,11 +186,17 @@ public abstract class SemanticNode { private static SqlWriterConfig getSqlWriterConfig(EngineType engineType) { SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); - SqlWriterConfig config = SqlPrettyWriter.config().withDialect(sqlDialect) - .withKeywordsLowerCase(false).withClauseEndsLine(true).withAlwaysUseParentheses(false) - .withSelectListItemsOnSeparateLines(false).withUpdateSetListNewline(false).withIndentation(0); + SqlWriterConfig config = + SqlPrettyWriter.config() + .withDialect(sqlDialect) + .withKeywordsLowerCase(false) + .withClauseEndsLine(true) + .withAlwaysUseParentheses(false) + .withSelectListItemsOnSeparateLines(false) + .withUpdateSetListNewline(false) + .withIndentation(0); if (EngineType.MYSQL.equals(engineType)) { - //no backticks around function name + // no backticks around function name config = config.withQuoteAllIdentifiers(false); } return config; @@ -191,8 +211,10 @@ public abstract class SemanticNode { case AS: SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode; if (sqlBasicCall.getOperandList().get(0).getKind().equals(SqlKind.IDENTIFIER)) { - addTableName(sqlBasicCall.getOperandList().get(0).toString(), - sqlBasicCall.getOperandList().get(1).toString(), parseInfo); + addTableName( + sqlBasicCall.getOperandList().get(0).toString(), + sqlBasicCall.getOperandList().get(1).toString(), + parseInfo); } else { sqlVisit(sqlBasicCall.getOperandList().get(0), parseInfo); } @@ -203,13 +225,17 @@ public abstract class SemanticNode { sqlVisit(sqlJoin.getRight(), parseInfo); SqlBasicCall condition = (SqlBasicCall) sqlJoin.getCondition(); if (Objects.nonNull(condition)) { - condition.getOperandList().stream().forEach(c -> addTagField(c.toString(), parseInfo, "")); + condition.getOperandList().stream() + .forEach(c -> addTagField(c.toString(), parseInfo, "")); } break; case UNION: - ((SqlBasicCall) sqlNode).getOperandList().forEach(node -> { - sqlVisit(node, parseInfo); - }); + ((SqlBasicCall) sqlNode) + .getOperandList() + .forEach( + node -> { + sqlVisit(node, parseInfo); + }); break; case WITH: SqlWith sqlWith = (SqlWith) sqlNode; @@ -226,9 +252,12 @@ public abstract class SemanticNode { } SqlSelect sqlSelect = (SqlSelect) select; SqlNodeList selectList = sqlSelect.getSelectList(); - selectList.getList().forEach(list -> { - fieldVisit(list, parseInfo, ""); - }); + selectList + .getList() + .forEach( + list -> { + fieldVisit(list, parseInfo, ""); + }); fromVisit(sqlSelect.getFrom(), parseInfo); if (sqlSelect.hasWhere()) { whereVisit((SqlBasicCall) sqlSelect.getWhere(), parseInfo); @@ -238,16 +267,17 @@ public abstract class SemanticNode { } SqlNodeList group = sqlSelect.getGroup(); if (group != null) { - group.forEach(groupField -> { - if (groupHints.contains(groupField.toString())) { - int groupIdx = Integer.valueOf(groupField.toString()) - 1; - if (selectList.getList().size() > groupIdx) { - fieldVisit(selectList.get(groupIdx), parseInfo, ""); - } - } else { - fieldVisit(groupField, parseInfo, ""); - } - }); + group.forEach( + groupField -> { + if (groupHints.contains(groupField.toString())) { + int groupIdx = Integer.valueOf(groupField.toString()) - 1; + if (selectList.getList().size() > groupIdx) { + fieldVisit(selectList.get(groupIdx), parseInfo, ""); + } + } else { + fieldVisit(groupField, parseInfo, ""); + } + }); } } @@ -255,7 +285,8 @@ public abstract class SemanticNode { if (where == null) { return; } - if (where.operandCount() == 2 && where.operand(0).getKind().equals(SqlKind.IDENTIFIER) + if (where.operandCount() == 2 + && where.operand(0).getKind().equals(SqlKind.IDENTIFIER) && where.operand(1).getKind().equals(SqlKind.LITERAL)) { fieldVisit(where.operand(0), parseInfo, ""); return; @@ -263,21 +294,21 @@ public abstract class SemanticNode { // 子查询 if (where.operandCount() == 2 && (where.operand(0).getKind().equals(SqlKind.IDENTIFIER) - && (where.operand(1).getKind().equals(SqlKind.SELECT) - || where.operand(1).getKind().equals(SqlKind.ORDER_BY))) - ) { + && (where.operand(1).getKind().equals(SqlKind.SELECT) + || where.operand(1).getKind().equals(SqlKind.ORDER_BY)))) { fieldVisit(where.operand(0), parseInfo, ""); sqlVisit((SqlNode) (where.operand(1)), parseInfo); return; } - if (CollectionUtils.isNotEmpty(where.getOperandList()) && where.operand(0).getKind() - .equals(SqlKind.IDENTIFIER)) { + if (CollectionUtils.isNotEmpty(where.getOperandList()) + && where.operand(0).getKind().equals(SqlKind.IDENTIFIER)) { fieldVisit(where.operand(0), parseInfo, ""); } if (where.operandCount() >= 2 && where.operand(1).getKind().equals(SqlKind.IDENTIFIER)) { fieldVisit(where.operand(1), parseInfo, ""); } - if (CollectionUtils.isNotEmpty(where.getOperandList()) && where.operand(0) instanceof SqlBasicCall) { + if (CollectionUtils.isNotEmpty(where.getOperandList()) + && where.operand(0) instanceof SqlBasicCall) { whereVisit(where.operand(0), parseInfo); } if (where.operandCount() >= 2 && where.operand(1) instanceof SqlBasicCall) { @@ -290,7 +321,7 @@ public abstract class SemanticNode { return; } SqlKind kind = field.getKind(); - //System.out.println(kind); + // System.out.println(kind); // aggfunction if (AGGREGATION_KIND.contains(kind)) { SqlOperator sqlCall = ((SqlCall) field).getOperator(); @@ -319,9 +350,12 @@ public abstract class SemanticNode { } } if (field instanceof SqlNodeList) { - ((SqlNodeList) field).getList().forEach(node -> { - fieldVisit(node, parseInfo, ""); - }); + ((SqlNodeList) field) + .getList() + .forEach( + node -> { + fieldVisit(node, parseInfo, ""); + }); } } @@ -330,7 +364,8 @@ public abstract class SemanticNode { if (!parseInfo.containsKey(Constants.SQL_PARSER_FIELD)) { parseInfo.put(Constants.SQL_PARSER_FIELD, new HashMap<>()); } - Map> fields = (Map>) parseInfo.get(Constants.SQL_PARSER_FIELD); + Map> fields = + (Map>) parseInfo.get(Constants.SQL_PARSER_FIELD); if (Pattern.matches("(?i)[a-z\\d_\\.]+", exp)) { if (exp.contains(".")) { @@ -345,9 +380,7 @@ public abstract class SemanticNode { } fields.get("").add(exp); } - } - } private static void fromVisit(SqlNode from, Map parseInfo) { @@ -374,7 +407,8 @@ public abstract class SemanticNode { sqlVisit(sqlJoin.getRight(), parseInfo); SqlBasicCall condition = (SqlBasicCall) sqlJoin.getCondition(); if (Objects.nonNull(condition)) { - condition.getOperandList().stream().forEach(c -> addTagField(c.toString(), parseInfo, "")); + condition.getOperandList().stream() + .forEach(c -> addTagField(c.toString(), parseInfo, "")); } break; case SELECT: @@ -392,12 +426,12 @@ public abstract class SemanticNode { if (!parseInfo.containsKey(Constants.SQL_PARSER_TABLE)) { parseInfo.put(Constants.SQL_PARSER_TABLE, new HashMap<>()); } - Map> dbTbs = (Map>) parseInfo.get(Constants.SQL_PARSER_TABLE); + Map> dbTbs = + (Map>) parseInfo.get(Constants.SQL_PARSER_TABLE); if (!dbTbs.containsKey(alias)) { dbTbs.put(alias, new HashSet<>()); } dbTbs.get(alias).add(exp); - } public static Map getDbTable(SqlNode sqlNode) { @@ -406,21 +440,28 @@ public abstract class SemanticNode { return parseInfo; } - public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode, - EngineType engineType) { + public static SqlNode optimize( + SqlValidatorScope scope, + SemanticSchema schema, + SqlNode sqlNode, + EngineType engineType) { try { HepProgramBuilder hepProgramBuilder = new HepProgramBuilder(); SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); - hepProgramBuilder.addRuleInstance(new FilterToGroupScanRule(FilterToGroupScanRule.DEFAULT, schema)); + hepProgramBuilder.addRuleInstance( + new FilterToGroupScanRule(FilterToGroupScanRule.DEFAULT, schema)); RelOptPlanner relOptPlanner = new HepPlanner(hepProgramBuilder.build()); RelToSqlConverter converter = new RelToSqlConverter(sqlDialect); - SqlValidator sqlValidator = Configuration.getSqlValidator( - scope.getValidator().getCatalogReader().getRootSchema(), engineType); - SqlToRelConverter sqlToRelConverter = Configuration.getSqlToRelConverter(scope, sqlValidator, - relOptPlanner, engineType); - RelNode sqlRel = sqlToRelConverter.convertQuery( - sqlValidator.validate(sqlNode), false, true).rel; - log.debug("RelNode optimize {}", + SqlValidator sqlValidator = + Configuration.getSqlValidator( + scope.getValidator().getCatalogReader().getRootSchema(), engineType); + SqlToRelConverter sqlToRelConverter = + Configuration.getSqlToRelConverter( + scope, sqlValidator, relOptPlanner, engineType); + RelNode sqlRel = + sqlToRelConverter.convertQuery(sqlValidator.validate(sqlNode), false, true).rel; + log.debug( + "RelNode optimize {}", SemanticNode.getSql(converter.visitRoot(sqlRel).asStatement(), engineType)); relOptPlanner.setRoot(sqlRel); RelNode relNode = relOptPlanner.findBestExp(); @@ -470,5 +511,4 @@ public abstract class SemanticNode { } return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java index 35b13655f..d51847e7a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java @@ -1,9 +1,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node.extend; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.ExtendNode; -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; @@ -13,9 +10,11 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWriter; -/** - * extend node to handle lateral explode dataSet - */ +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; + +/** extend node to handle lateral explode dataSet */ public class LateralViewExplodeNode extends ExtendNode { public final String sqlNameView = "view"; @@ -61,8 +60,9 @@ public class LateralViewExplodeNode extends ExtendNode { public void explode(SqlWriter writer, SqlNode sqlNode) { String delimiter = - Objects.nonNull(delimiterMap) && delimiterMap.containsKey(sqlNode.toString()) ? delimiterMap.get( - sqlNode.toString()) : ""; + Objects.nonNull(delimiterMap) && delimiterMap.containsKey(sqlNode.toString()) + ? delimiterMap.get(sqlNode.toString()) + : ""; if (delimiter.isEmpty()) { writer.sep(sqlNameExplode); } else { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java index 272b3ba4a..ed77131a8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java @@ -1,12 +1,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; import org.apache.calcite.rel.core.Aggregate; @@ -26,23 +20,39 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; import org.apache.commons.lang3.tuple.Triple; -/** - * push down the time filter into group using the RuntimeOptions defined minMaxTime - * - */ -public class FilterToGroupScanRule extends RelRule - implements TransformationRule { +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; - public static FilterTableScanRule.Config DEFAULT = FilterTableScanRule.Config.DEFAULT.withOperandSupplier((b0) -> { - return b0.operand(LogicalFilter.class).oneInput((b1) -> { - return b1.operand(LogicalProject.class).oneInput((b2) -> { - return b2.operand(LogicalAggregate.class).oneInput((b3) -> { - return b3.operand(LogicalProject.class).anyInputs(); - }); - }); - }); +/** push down the time filter into group using the RuntimeOptions defined minMaxTime */ +public class FilterToGroupScanRule extends RelRule implements TransformationRule { - }).as(FilterTableScanRule.Config.class); + public static FilterTableScanRule.Config DEFAULT = + FilterTableScanRule.Config.DEFAULT + .withOperandSupplier( + (b0) -> { + return b0.operand(LogicalFilter.class) + .oneInput( + (b1) -> { + return b1.operand(LogicalProject.class) + .oneInput( + (b2) -> { + return b2.operand( + LogicalAggregate + .class) + .oneInput( + (b3) -> { + return b3.operand( + LogicalProject + .class) + .anyInputs(); + }); + }); + }); + }) + .as(FilterTableScanRule.Config.class); private SemanticSchema semanticSchema; @@ -55,9 +65,9 @@ public class FilterToGroupScanRule extends RelRule if (call.rels.length != 4) { return; } - if (Objects.isNull(semanticSchema.getRuntimeOptions()) || Objects.isNull( - semanticSchema.getRuntimeOptions().getMinMaxTime()) || semanticSchema.getRuntimeOptions() - .getMinMaxTime().getLeft().isEmpty()) { + if (Objects.isNull(semanticSchema.getRuntimeOptions()) + || Objects.isNull(semanticSchema.getRuntimeOptions().getMinMaxTime()) + || semanticSchema.getRuntimeOptions().getMinMaxTime().getLeft().isEmpty()) { return; } Triple minMax = semanticSchema.getRuntimeOptions().getMinMaxTime(); @@ -65,17 +75,20 @@ public class FilterToGroupScanRule extends RelRule Project project0 = (Project) call.rel(1); Project project1 = (Project) call.rel(3); Aggregate logicalAggregate = (Aggregate) call.rel(2); - Optional> isIn = project1.getNamedProjects() - .stream().filter(i -> i.right.equalsIgnoreCase(minMax.getLeft())).findFirst(); + Optional> isIn = + project1.getNamedProjects().stream() + .filter(i -> i.right.equalsIgnoreCase(minMax.getLeft())) + .findFirst(); if (!isIn.isPresent()) { return; } RelBuilder relBuilder = call.builder(); relBuilder.push(project1); - RexNode addPartitionCondition = getRexNodeByTimeRange(relBuilder, minMax.getLeft(), minMax.getMiddle(), - minMax.getRight()); - relBuilder.filter(new RexNode[]{addPartitionCondition}); + RexNode addPartitionCondition = + getRexNodeByTimeRange( + relBuilder, minMax.getLeft(), minMax.getMiddle(), minMax.getRight()); + relBuilder.filter(new RexNode[] {addPartitionCondition}); relBuilder.project(project1.getProjects()); ImmutableBitSet newGroupSet = logicalAggregate.getGroupSet(); int newGroupCount = newGroupSet.cardinality(); @@ -85,20 +98,30 @@ public class FilterToGroupScanRule extends RelRule while (var.hasNext()) { AggregateCall aggCall = (AggregateCall) var.next(); newAggCalls.add( - aggCall.adaptTo(project1, aggCall.getArgList(), aggCall.filterArg, groupCount, newGroupCount)); + aggCall.adaptTo( + project1, + aggCall.getArgList(), + aggCall.filterArg, + groupCount, + newGroupCount)); } relBuilder.aggregate(relBuilder.groupKey(newGroupSet), newAggCalls); relBuilder.project(project0.getProjects()); - relBuilder.filter(new RexNode[]{filter.getCondition()}); + relBuilder.filter(new RexNode[] {filter.getCondition()}); call.transformTo(relBuilder.build()); } - private RexNode getRexNodeByTimeRange(RelBuilder relBuilder, String dateField, String start, String end) { - return relBuilder.call(SqlStdOperatorTable.AND, - relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, relBuilder.field(dateField), + private RexNode getRexNodeByTimeRange( + RelBuilder relBuilder, String dateField, String start, String end) { + return relBuilder.call( + SqlStdOperatorTable.AND, + relBuilder.call( + SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + relBuilder.field(dateField), relBuilder.literal(start)), - relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(dateField), + relBuilder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + relBuilder.field(dateField), relBuilder.literal(end))); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java index 62becb7fa..adcb1907d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java @@ -1,17 +1,21 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -19,24 +23,23 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -/** - * process query specified filtering information - */ +/** process query specified filtering information */ public class FilterRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + public void render( + MetricQueryParam metricCommand, + List dataSources, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception { TableView tableView = super.tableView; SqlNode filterNode = null; List queryMetrics = new ArrayList<>(metricCommand.getMetrics()); List queryDimensions = new ArrayList<>(metricCommand.getDimensions()); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); if (metricCommand.getWhere() != null && !metricCommand.getWhere().isEmpty()) { filterNode = SemanticNode.parse(metricCommand.getWhere(), scope, engineType); @@ -46,8 +49,14 @@ public class FilterRender extends Renderer { Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); for (DataSource dataSource : dataSources) { - SourceRender.whereDimMetric(fieldWhere, metricCommand.getMetrics(), metricCommand.getDimensions(), - dataSource, schema, dimensions, metrics); + SourceRender.whereDimMetric( + fieldWhere, + metricCommand.getMetrics(), + metricCommand.getDimensions(), + dataSource, + schema, + dimensions, + metrics); } queryMetrics.addAll(metrics); queryDimensions.addAll(dimensions); @@ -62,14 +71,18 @@ public class FilterRender extends Renderer { continue; } if (optionalMetric.isPresent()) { - tableView.getMeasure().add(MetricNode.build(optionalMetric.get(), scope, engineType)); + tableView + .getMeasure() + .add(MetricNode.build(optionalMetric.get(), scope, engineType)); } else { tableView.getMeasure().add(SemanticNode.parse(metric, scope, engineType)); } } if (filterNode != null) { TableView filterView = new TableView(); - filterView.setTable(SemanticNode.buildAs(Constants.DATASOURCE_TABLE_FILTER_PREFIX, tableView.build())); + filterView.setTable( + SemanticNode.buildAs( + Constants.DATASOURCE_TABLE_FILTER_PREFIX, tableView.build())); filterView.getFilter().add(filterNode); filterView.getMeasure().add(SqlIdentifier.star(SqlParserPos.ZERO)); super.tableView = filterView; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java index 0252f3da8..989407941 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java @@ -2,14 +2,14 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode; @@ -43,17 +43,21 @@ import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; -/** - * process the join conditions when the source number is greater than 1 - */ +/** process the join conditions when the source number is greater than 1 */ @Slf4j public class JoinRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + public void render( + MetricQueryParam metricCommand, + List dataSources, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception { String queryWhere = metricCommand.getWhere(); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); if (queryWhere != null && !queryWhere.isEmpty()) { @@ -78,7 +82,13 @@ public class JoinRender extends Renderer { final Set filterMetrics = new HashSet<>(); final List queryDimension = new ArrayList<>(); final List queryMetrics = new ArrayList<>(); - SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataSource, schema, filterDimensions, + SourceRender.whereDimMetric( + fieldWhere, + queryMetrics, + queryDimension, + dataSource, + schema, + filterDimensions, filterMetrics); List reqMetric = new ArrayList<>(metricCommand.getMetrics()); reqMetric.addAll(filterMetrics); @@ -88,13 +98,32 @@ public class JoinRender extends Renderer { reqDimension.addAll(filterDimensions); reqDimension = uniqList(reqDimension); - Set sourceMeasure = dataSource.getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); - doMetric(innerSelect, filterView, queryMetrics, reqMetric, dataSource, sourceMeasure, scope, schema, + Set sourceMeasure = + dataSource.getMeasures().stream() + .map(mm -> mm.getName()) + .collect(Collectors.toSet()); + doMetric( + innerSelect, + filterView, + queryMetrics, + reqMetric, + dataSource, + sourceMeasure, + scope, + schema, nonAgg); - Set dimension = dataSource.getDimensions().stream().map(dd -> dd.getName()) - .collect(Collectors.toSet()); - doDimension(innerSelect, filterDimension, queryDimension, reqDimension, dataSource, dimension, scope, + Set dimension = + dataSource.getDimensions().stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); + doDimension( + innerSelect, + filterDimension, + queryDimension, + reqDimension, + dataSource, + dimension, + scope, schema); List primary = new ArrayList<>(); for (Identify identify : dataSource.getIdentifiers()) { @@ -105,8 +134,17 @@ public class JoinRender extends Renderer { } List dataSourceWhere = new ArrayList<>(fieldWhere); addZipperField(dataSource, dataSourceWhere); - TableView tableView = SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension, - metricCommand.getWhere(), dataSources.get(i), scope, schema, true); + TableView tableView = + SourceRender.renderOne( + "", + dataSourceWhere, + queryMetrics, + queryDimension, + metricCommand.getWhere(), + dataSources.get(i), + scope, + schema, + true); log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString())); String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); tableView.setAlias(alias); @@ -127,7 +165,8 @@ public class JoinRender extends Renderer { innerView.getMeasure().add(entry.getValue()); } innerView.setTable(left); - filterView.setTable(SemanticNode.buildAs(Constants.JOIN_TABLE_OUT_PREFIX, innerView.build())); + filterView.setTable( + SemanticNode.buildAs(Constants.JOIN_TABLE_OUT_PREFIX, innerView.build())); if (!filterDimension.isEmpty()) { for (String d : getQueryDimension(filterDimension, queryAllDimension, whereFields)) { if (nonAgg) { @@ -135,83 +174,127 @@ public class JoinRender extends Renderer { } else { filterView.getDimension().add(SemanticNode.parse(d, scope, engineType)); } - } } super.tableView = filterView; } - private void doMetric(Map innerSelect, TableView filterView, List queryMetrics, - List reqMetrics, DataSource dataSource, Set sourceMeasure, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + private void doMetric( + Map innerSelect, + TableView filterView, + List queryMetrics, + List reqMetrics, + DataSource dataSource, + Set sourceMeasure, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); for (String m : reqMetrics) { if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) { - MetricNode metricNode = buildMetricNode(m, dataSource, scope, schema, nonAgg, alias); + MetricNode metricNode = + buildMetricNode(m, dataSource, scope, schema, nonAgg, alias); if (!metricNode.getNonAggNode().isEmpty()) { for (String measure : metricNode.getNonAggNode().keySet()) { - innerSelect.put(measure, - SemanticNode.buildAs(measure, - SemanticNode.parse(alias + "." + measure, scope, engineType))); + innerSelect.put( + measure, + SemanticNode.buildAs( + measure, + SemanticNode.parse( + alias + "." + measure, scope, engineType))); } - } if (metricNode.getAggFunction() != null && !metricNode.getAggFunction().isEmpty()) { for (Map.Entry entry : metricNode.getAggFunction().entrySet()) { if (metricNode.getNonAggNode().containsKey(entry.getKey())) { if (nonAgg) { - filterView.getMeasure().add(SemanticNode.buildAs(entry.getKey(), - SemanticNode.parse(entry.getKey(), scope, engineType))); + filterView + .getMeasure() + .add( + SemanticNode.buildAs( + entry.getKey(), + SemanticNode.parse( + entry.getKey(), + scope, + engineType))); } else { - filterView.getMeasure().add(SemanticNode.buildAs(entry.getKey(), - AggFunctionNode.build(entry.getValue(), entry.getKey(), scope, engineType))); + filterView + .getMeasure() + .add( + SemanticNode.buildAs( + entry.getKey(), + AggFunctionNode.build( + entry.getValue(), + entry.getKey(), + scope, + engineType))); } - } } - } } } } - private void doDimension(Map innerSelect, Set filterDimension, List queryDimension, - List reqDimensions, DataSource dataSource, Set dimension, SqlValidatorScope scope, - SemanticSchema schema) throws Exception { + private void doDimension( + Map innerSelect, + Set filterDimension, + List queryDimension, + List reqDimensions, + DataSource dataSource, + Set dimension, + SqlValidatorScope scope, + SemanticSchema schema) + throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); for (String d : reqDimensions) { if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) { if (d.contains(Constants.DIMENSION_IDENTIFY)) { String[] identifyDimension = d.split(Constants.DIMENSION_IDENTIFY); - innerSelect.put(d, - SemanticNode.buildAs(d, - SemanticNode.parse(alias + "." + identifyDimension[1], scope, engineType))); + innerSelect.put( + d, + SemanticNode.buildAs( + d, + SemanticNode.parse( + alias + "." + identifyDimension[1], + scope, + engineType))); } else { - innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode.parse(alias + "." + d, scope, engineType))); - + innerSelect.put( + d, + SemanticNode.buildAs( + d, SemanticNode.parse(alias + "." + d, scope, engineType))); } filterDimension.add(d); } } } - private Set getQueryDimension(Set filterDimension, Set queryAllDimension, - Set whereFields) { - return filterDimension.stream().filter(d -> queryAllDimension.contains(d) || whereFields.contains(d)).collect( - Collectors.toSet()); + private Set getQueryDimension( + Set filterDimension, Set queryAllDimension, Set whereFields) { + return filterDimension.stream() + .filter(d -> queryAllDimension.contains(d) || whereFields.contains(d)) + .collect(Collectors.toSet()); } - private boolean getMatchMetric(SemanticSchema schema, Set sourceMeasure, String m, - List queryMetrics) { - Optional metric = schema.getMetrics().stream().filter(mm -> mm.getName().equalsIgnoreCase(m)) - .findFirst(); + private boolean getMatchMetric( + SemanticSchema schema, Set sourceMeasure, String m, List queryMetrics) { + Optional metric = + schema.getMetrics().stream() + .filter(mm -> mm.getName().equalsIgnoreCase(m)) + .findFirst(); boolean isAdd = false; if (metric.isPresent()) { - Set metricMeasures = metric.get().getMetricTypeParams().getMeasures().stream() - .map(me -> me.getName()).collect(Collectors.toSet()); + Set metricMeasures = + metric.get().getMetricTypeParams().getMeasures().stream() + .map(me -> me.getName()) + .collect(Collectors.toSet()); if (sourceMeasure.containsAll(metricMeasures)) { isAdd = true; } @@ -225,8 +308,12 @@ public class JoinRender extends Renderer { return isAdd; } - private boolean getMatchDimension(SemanticSchema schema, Set sourceDimension, DataSource dataSource, - String d, List queryDimension) { + private boolean getMatchDimension( + SemanticSchema schema, + Set sourceDimension, + DataSource dataSource, + String d, + List queryDimension) { String oriDimension = d; boolean isAdd = false; if (d.contains(Constants.DIMENSION_IDENTIFY)) { @@ -258,11 +345,19 @@ public class JoinRender extends Renderer { return SemanticNode.getTable(tableView.getTable()); } - private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map before, - DataSource dataSource, SemanticSchema schema, SqlValidatorScope scope) + private SqlNode buildJoin( + SqlNode left, + TableView leftTable, + TableView tableView, + Map before, + DataSource dataSource, + SemanticSchema schema, + SqlValidatorScope scope) throws Exception { - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); - SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope, engineType); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + SqlNode condition = + getCondition(leftTable, tableView, dataSource, schema, scope, engineType); SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema); SqlNode joinRelationCondition = null; @@ -272,13 +367,18 @@ public class JoinRender extends Renderer { condition = joinRelationCondition; } if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType()) - || Materialization.TimePartType.ZIPPER.equals(tableView.getDataSource().getTimePartType())) { - SqlNode zipperCondition = getZipperCondition(leftTable, tableView, dataSource, schema, scope); + || Materialization.TimePartType.ZIPPER.equals( + tableView.getDataSource().getTimePartType())) { + SqlNode zipperCondition = + getZipperCondition(leftTable, tableView, dataSource, schema, scope); if (Objects.nonNull(joinRelationCondition)) { - condition = new SqlBasicCall( - SqlStdOperatorTable.AND, - new ArrayList<>(Arrays.asList(zipperCondition, joinRelationCondition)), - SqlParserPos.ZERO, null); + condition = + new SqlBasicCall( + SqlStdOperatorTable.AND, + new ArrayList<>( + Arrays.asList(zipperCondition, joinRelationCondition)), + SqlParserPos.ZERO, + null); } else { condition = zipperCondition; } @@ -291,29 +391,38 @@ public class JoinRender extends Renderer { sqlLiteral, SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)), SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO), - condition - ); + condition); } - private JoinRelation getMatchJoinRelation(Map before, TableView tableView, SemanticSchema schema) { + private JoinRelation getMatchJoinRelation( + Map before, TableView tableView, SemanticSchema schema) { JoinRelation matchJoinRelation = JoinRelation.builder().build(); if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { for (JoinRelation joinRelation : schema.getJoinRelations()) { if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataSource().getName()) && before.containsKey(joinRelation.getLeft())) { - matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() - .map(r -> Triple.of(before.get(joinRelation.getLeft()) + "." + r.getLeft(), - r.getMiddle(), tableView.getAlias() + "." + r.getRight())).collect( - Collectors.toList())); + matchJoinRelation.setJoinCondition( + joinRelation.getJoinCondition().stream() + .map( + r -> + Triple.of( + before.get(joinRelation.getLeft()) + + "." + + r.getLeft(), + r.getMiddle(), + tableView.getAlias() + + "." + + r.getRight())) + .collect(Collectors.toList())); matchJoinRelation.setJoinType(joinRelation.getJoinType()); } } - } return matchJoinRelation; } - private SqlNode getCondition(JoinRelation joinRelation, SqlValidatorScope scope, EngineType engineType) + private SqlNode getCondition( + JoinRelation joinRelation, SqlValidatorScope scope, EngineType engineType) throws Exception { SqlNode condition = null; for (Triple con : joinRelation.getJoinCondition()) { @@ -321,26 +430,38 @@ public class JoinRender extends Renderer { ons.add(SemanticNode.parse(con.getLeft(), scope, engineType)); ons.add(SemanticNode.parse(con.getRight(), scope, engineType)); if (Objects.isNull(condition)) { - condition = new SqlBasicCall( - SemanticNode.getBinaryOperator(con.getMiddle()), - ons, - SqlParserPos.ZERO, null); + condition = + new SqlBasicCall( + SemanticNode.getBinaryOperator(con.getMiddle()), + ons, + SqlParserPos.ZERO, + null); continue; } - SqlNode addCondition = new SqlBasicCall( - SemanticNode.getBinaryOperator(con.getMiddle()), - ons, - SqlParserPos.ZERO, null); - condition = new SqlBasicCall( - SqlStdOperatorTable.AND, - new ArrayList<>(Arrays.asList(condition, addCondition)), - SqlParserPos.ZERO, null); + SqlNode addCondition = + new SqlBasicCall( + SemanticNode.getBinaryOperator(con.getMiddle()), + ons, + SqlParserPos.ZERO, + null); + condition = + new SqlBasicCall( + SqlStdOperatorTable.AND, + new ArrayList<>(Arrays.asList(condition, addCondition)), + SqlParserPos.ZERO, + null); } return condition; } - private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, - SqlValidatorScope scope, EngineType engineType) throws Exception { + private SqlNode getCondition( + TableView left, + TableView right, + DataSource dataSource, + SemanticSchema schema, + SqlValidatorScope scope, + EngineType engineType) + throws Exception { Set selectLeft = SemanticNode.getSelect(left.getTable()); Set selectRight = SemanticNode.getSelect(right.getTable()); @@ -364,25 +485,27 @@ public class JoinRender extends Renderer { ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope, engineType)); ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope, engineType)); if (condition == null) { - condition = new SqlBasicCall( - SqlStdOperatorTable.EQUALS, - ons, - SqlParserPos.ZERO, null); + condition = + new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null); continue; } - SqlNode addCondition = new SqlBasicCall( - SqlStdOperatorTable.EQUALS, - ons, - SqlParserPos.ZERO, null); - condition = new SqlBasicCall( - SqlStdOperatorTable.AND, - new ArrayList<>(Arrays.asList(condition, addCondition)), - SqlParserPos.ZERO, null); + SqlNode addCondition = + new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null); + condition = + new SqlBasicCall( + SqlStdOperatorTable.AND, + new ArrayList<>(Arrays.asList(condition, addCondition)), + SqlParserPos.ZERO, + null); } return condition; } - private static void joinOrder(int cnt, String id, Map> next, Queue orders, + private static void joinOrder( + int cnt, + String id, + Map> next, + Queue orders, Map visited) { visited.put(id, true); orders.add(id); @@ -404,32 +527,42 @@ public class JoinRender extends Renderer { private void addZipperField(DataSource dataSource, List fields) { if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { dataSource.getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).forEach(t -> { - if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END) - && !fields.contains(t.getName()) - ) { - fields.add(t.getName()); - } - if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START) - && !fields.contains(t.getName()) - ) { - fields.add(t.getName()); - } - }); + .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) + .forEach( + t -> { + if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END) + && !fields.contains(t.getName())) { + fields.add(t.getName()); + } + if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START) + && !fields.contains(t.getName())) { + fields.add(t.getName()); + } + }); } } - private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, - SqlValidatorScope scope) throws Exception { + private SqlNode getZipperCondition( + TableView left, + TableView right, + DataSource dataSource, + SemanticSchema schema, + SqlValidatorScope scope) + throws Exception { if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) - && Materialization.TimePartType.ZIPPER.equals(right.getDataSource().getTimePartType())) { + && Materialization.TimePartType.ZIPPER.equals( + right.getDataSource().getTimePartType())) { throw new Exception("not support two zipper table"); } SqlNode condition = null; - Optional leftTime = left.getDataSource().getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst(); - Optional rightTime = right.getDataSource().getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst(); + Optional leftTime = + left.getDataSource().getDimensions().stream() + .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) + .findFirst(); + Optional rightTime = + right.getDataSource().getDimensions().stream() + .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) + .findFirst(); if (leftTime.isPresent() && rightTime.isPresent()) { String startTime = ""; @@ -437,47 +570,92 @@ public class JoinRender extends Renderer { String dateTime = ""; Optional startTimeOp = - (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) ? left : right) + (Materialization.TimePartType.ZIPPER.equals( + left.getDataSource().getTimePartType()) + ? left + : right) .getDataSource().getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) - .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START)).findFirst(); + .filter( + d -> + Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase( + d.getType())) + .filter( + d -> + d.getName() + .startsWith( + Constants + .MATERIALIZATION_ZIPPER_START)) + .findFirst(); Optional endTimeOp = - (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) ? left : right) + (Materialization.TimePartType.ZIPPER.equals( + left.getDataSource().getTimePartType()) + ? left + : right) .getDataSource().getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) - .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)).findFirst(); + .filter( + d -> + Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase( + d.getType())) + .filter( + d -> + d.getName() + .startsWith( + Constants + .MATERIALIZATION_ZIPPER_END)) + .findFirst(); if (startTimeOp.isPresent() && endTimeOp.isPresent()) { TableView zipper = - Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) - ? left : right; + Materialization.TimePartType.ZIPPER.equals( + left.getDataSource().getTimePartType()) + ? left + : right; TableView partMetric = - Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) - ? right : left; + Materialization.TimePartType.ZIPPER.equals( + left.getDataSource().getTimePartType()) + ? right + : left; Optional partTime = - Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) - ? rightTime : leftTime; + Materialization.TimePartType.ZIPPER.equals( + left.getDataSource().getTimePartType()) + ? rightTime + : leftTime; startTime = zipper.getAlias() + "." + startTimeOp.get().getName(); endTime = zipper.getAlias() + "." + endTimeOp.get().getName(); dateTime = partMetric.getAlias() + "." + partTime.get().getName(); } - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); - ArrayList operandList = new ArrayList<>( - Arrays.asList(SemanticNode.parse(endTime, scope, engineType), - SemanticNode.parse(dateTime, scope, engineType))); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + ArrayList operandList = + new ArrayList<>( + Arrays.asList( + SemanticNode.parse(endTime, scope, engineType), + SemanticNode.parse(dateTime, scope, engineType))); condition = new SqlBasicCall( SqlStdOperatorTable.AND, - new ArrayList(Arrays.asList(new SqlBasicCall( - SqlStdOperatorTable.LESS_THAN_OR_EQUAL, - new ArrayList( - Arrays.asList(SemanticNode.parse(startTime, scope, engineType), - SemanticNode.parse(dateTime, scope, engineType))), - SqlParserPos.ZERO, null), new SqlBasicCall( - SqlStdOperatorTable.GREATER_THAN, - operandList, - SqlParserPos.ZERO, null))), - SqlParserPos.ZERO, null); - + new ArrayList( + Arrays.asList( + new SqlBasicCall( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + new ArrayList( + Arrays.asList( + SemanticNode.parse( + startTime, + scope, + engineType), + SemanticNode.parse( + dateTime, + scope, + engineType))), + SqlParserPos.ZERO, + null), + new SqlBasicCall( + SqlStdOperatorTable.GREATER_THAN, + operandList, + SqlParserPos.ZERO, + null))), + SqlParserPos.ZERO, + null); } return condition; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java index 9a49e95d7..3496157eb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java @@ -1,18 +1,14 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; - +import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.common.pojo.ColumnOrder; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; -import java.util.ArrayList; -import java.util.List; - import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.fun.SqlStdOperatorTable; @@ -20,16 +16,23 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.springframework.util.CollectionUtils; -/** - * process the query result items from query request - */ +import java.util.ArrayList; +import java.util.List; + +/** process the query result items from query request */ public class OutputRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + public void render( + MetricQueryParam metricCommand, + List dataSources, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception { TableView selectDataSet = super.tableView; - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); for (String dimension : metricCommand.getDimensions()) { selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope, engineType)); } @@ -42,15 +45,20 @@ public class OutputRender extends Renderer { } if (metricCommand.getLimit() > 0) { - SqlNode offset = SemanticNode.parse(metricCommand.getLimit().toString(), scope, engineType); + SqlNode offset = + SemanticNode.parse(metricCommand.getLimit().toString(), scope, engineType); selectDataSet.setOffset(offset); } if (!CollectionUtils.isEmpty(metricCommand.getOrder())) { List orderList = new ArrayList<>(); for (ColumnOrder columnOrder : metricCommand.getOrder()) { if (SqlStdOperatorTable.DESC.getName().equalsIgnoreCase(columnOrder.getOrder())) { - orderList.add(SqlStdOperatorTable.DESC.createCall(SqlParserPos.ZERO, - new SqlNode[]{SemanticNode.parse(columnOrder.getCol(), scope, engineType)})); + orderList.add( + SqlStdOperatorTable.DESC.createCall( + SqlParserPos.ZERO, + new SqlNode[] { + SemanticNode.parse(columnOrder.getCol(), scope, engineType) + })); } else { orderList.add(SemanticNode.parse(columnOrder.getCol(), scope, engineType)); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java index 46b7df1e6..0c2d0b31b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java @@ -1,17 +1,15 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; - -import static com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants.DIMENSION_DELIMITER; - import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; @@ -20,7 +18,12 @@ import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNo import com.tencent.supersonic.headless.core.translator.calcite.sql.node.IdentifyNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.util.Litmus; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -33,22 +36,23 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.apache.calcite.util.Litmus; -import org.springframework.util.CollectionUtils; +import static com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants.DIMENSION_DELIMITER; -/** - * process the table dataSet from the defined data model schema - */ +/** process the table dataSet from the defined data model schema */ @Slf4j public class SourceRender extends Renderer { - public static TableView renderOne(String alias, List fieldWheres, - List reqMetrics, List reqDimensions, - String queryWhere, DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + public static TableView renderOne( + String alias, + List fieldWheres, + List reqMetrics, + List reqDimensions, + String queryWhere, + DataSource datasource, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception { TableView dataSet = new TableView(); TableView output = new TableView(); @@ -59,36 +63,63 @@ public class SourceRender extends Renderer { if (!fieldWhere.isEmpty()) { Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); - whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema, dimensions, metrics); + whereDimMetric( + fieldWhere, + queryMetrics, + queryDimensions, + datasource, + schema, + dimensions, + metrics); queryMetrics.addAll(metrics); queryMetrics = uniqList(queryMetrics); queryDimensions.addAll(dimensions); queryDimensions = uniqList(queryDimensions); - mergeWhere(fieldWhere, dataSet, output, queryMetrics, queryDimensions, extendFields, - datasource, scope, - schema, nonAgg); + mergeWhere( + fieldWhere, + dataSet, + output, + queryMetrics, + queryDimensions, + extendFields, + datasource, + scope, + schema, + nonAgg); } addTimeDimension(datasource, queryDimensions); for (String metric : queryMetrics) { - MetricNode metricNode = buildMetricNode(metric, datasource, scope, schema, nonAgg, alias); + MetricNode metricNode = + buildMetricNode(metric, datasource, scope, schema, nonAgg, alias); if (!metricNode.getAggNode().isEmpty()) { - metricNode.getAggNode().entrySet().stream().forEach(m -> output.getMeasure().add(m.getValue())); + metricNode.getAggNode().entrySet().stream() + .forEach(m -> output.getMeasure().add(m.getValue())); } if (metricNode.getNonAggNode() != null) { - metricNode.getNonAggNode().entrySet().stream().forEach(m -> dataSet.getMeasure().add(m.getValue())); + metricNode.getNonAggNode().entrySet().stream() + .forEach(m -> dataSet.getMeasure().add(m.getValue())); } if (metricNode.getMeasureFilter() != null) { - metricNode.getMeasureFilter().entrySet().stream().forEach(m -> dataSet.getFilter().add(m.getValue())); + metricNode.getMeasureFilter().entrySet().stream() + .forEach(m -> dataSet.getFilter().add(m.getValue())); } } for (String dimension : queryDimensions) { - if (dimension.contains(Constants.DIMENSION_IDENTIFY) && queryDimensions.contains( - dimension.split(Constants.DIMENSION_IDENTIFY)[1])) { + if (dimension.contains(Constants.DIMENSION_IDENTIFY) + && queryDimensions.contains(dimension.split(Constants.DIMENSION_IDENTIFY)[1])) { continue; } - buildDimension(dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "", - dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension.split(Constants.DIMENSION_IDENTIFY)[1] - : dimension, datasource, schema, nonAgg, extendFields, dataSet, output, + buildDimension( + dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "", + dimension.contains(Constants.DIMENSION_IDENTIFY) + ? dimension.split(Constants.DIMENSION_IDENTIFY)[1] + : dimension, + datasource, + schema, + nonAgg, + extendFields, + dataSet, + output, scope); } @@ -97,13 +128,17 @@ public class SourceRender extends Renderer { SqlNode tableNode = DataSourceNode.buildExtend(datasource, extendFields, scope); dataSet.setTable(tableNode); - output.setTable(SemanticNode.buildAs( - Constants.DATASOURCE_TABLE_OUT_PREFIX + datasource.getName() + "_" + UUID.randomUUID().toString() - .substring(32), dataSet.build())); + output.setTable( + SemanticNode.buildAs( + Constants.DATASOURCE_TABLE_OUT_PREFIX + + datasource.getName() + + "_" + + UUID.randomUUID().toString().substring(32), + dataSet.build())); return output; } - private static List deduplicateNode(List listNode) { //List去重 + private static List deduplicateNode(List listNode) { // List去重 List uniqueElements = new ArrayList<>(); for (SqlNode element : listNode) { if (!containsElement(uniqueElements, element)) { @@ -113,7 +148,8 @@ public class SourceRender extends Renderer { return uniqueElements; } - private static boolean containsElement(List list, SqlNode element) { //检查List中是否含有某element + private static boolean containsElement( + List list, SqlNode element) { // 检查List中是否含有某element for (SqlNode i : list) { if (i.equalsDeep(element, Litmus.IGNORE)) { return true; @@ -122,12 +158,20 @@ public class SourceRender extends Renderer { return false; } - private static void buildDimension(String alias, String dimension, DataSource datasource, SemanticSchema schema, - boolean nonAgg, Map extendFields, TableView dataSet, TableView output, + private static void buildDimension( + String alias, + String dimension, + DataSource datasource, + SemanticSchema schema, + boolean nonAgg, + Map extendFields, + TableView dataSet, + TableView output, SqlValidatorScope scope) throws Exception { List dimensionList = schema.getDimension().get(datasource.getName()); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); boolean isAdd = false; if (!CollectionUtils.isEmpty(dimensionList)) { for (Dimension dim : dimensionList) { @@ -145,22 +189,29 @@ public class SourceRender extends Renderer { if ("".equals(alias)) { output.getDimension().add(DimensionNode.buildName(dim, scope, engineType)); } else { - output.getDimension().add(DimensionNode.buildNameAs(alias, dim, scope, engineType)); + output.getDimension() + .add(DimensionNode.buildNameAs(alias, dim, scope, engineType)); } isAdd = true; break; } } if (!isAdd) { - Optional identify = datasource.getIdentifiers().stream() - .filter(i -> i.getName().equalsIgnoreCase(dimension)).findFirst(); + Optional identify = + datasource.getIdentifiers().stream() + .filter(i -> i.getName().equalsIgnoreCase(dimension)) + .findFirst(); if (identify.isPresent()) { if (nonAgg) { - dataSet.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); - output.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); + dataSet.getMeasure() + .add(SemanticNode.parse(identify.get().getName(), scope, engineType)); + output.getMeasure() + .add(SemanticNode.parse(identify.get().getName(), scope, engineType)); } else { - dataSet.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); - output.getDimension().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); + dataSet.getMeasure() + .add(SemanticNode.parse(identify.get().getName(), scope, engineType)); + output.getDimension() + .add(SemanticNode.parse(identify.get().getName(), scope, engineType)); } isAdd = true; } @@ -170,35 +221,45 @@ public class SourceRender extends Renderer { } Optional dimensionOptional = getDimensionByName(dimension, datasource); if (dimensionOptional.isPresent()) { - dataSet.getMeasure().add(DimensionNode.buildArray(dimensionOptional.get(), scope, engineType)); + dataSet.getMeasure() + .add(DimensionNode.buildArray(dimensionOptional.get(), scope, engineType)); addExtendFields(dimensionOptional.get(), extendFields); if (nonAgg) { - output.getMeasure().add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); + output.getMeasure() + .add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); return; } - output.getDimension().add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); + output.getDimension() + .add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); } } private static void addExtendFields(Dimension dimension, Map extendFields) { if (dimension.getDataType().isArray()) { - if (Objects.nonNull(dimension.getExt()) && dimension.getExt() - .containsKey(DIMENSION_DELIMITER)) { - extendFields.put(dimension.getExpr(), - (String) dimension.getExt().get(DIMENSION_DELIMITER)); + if (Objects.nonNull(dimension.getExt()) + && dimension.getExt().containsKey(DIMENSION_DELIMITER)) { + extendFields.put( + dimension.getExpr(), (String) dimension.getExt().get(DIMENSION_DELIMITER)); } else { extendFields.put(dimension.getExpr(), ""); } } } - private static List getWhereMeasure(List fields, List queryMetrics, - List queryDimensions, Map extendFields, DataSource datasource, + private static List getWhereMeasure( + List fields, + List queryMetrics, + List queryDimensions, + Map extendFields, + DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, + boolean nonAgg) + throws Exception { Iterator iterator = fields.iterator(); List whereNode = new ArrayList<>(); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); while (iterator.hasNext()) { String cur = iterator.next(); if (queryDimensions.contains(cur) || queryMetrics.contains(cur)) { @@ -234,21 +295,39 @@ public class SourceRender extends Renderer { return whereNode; } - private static void mergeWhere(List fields, TableView dataSet, TableView outputSet, + private static void mergeWhere( + List fields, + TableView dataSet, + TableView outputSet, List queryMetrics, - List queryDimensions, Map extendFields, DataSource datasource, + List queryDimensions, + Map extendFields, + DataSource datasource, SqlValidatorScope scope, SemanticSchema schema, - boolean nonAgg) throws Exception { - List whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource, - scope, schema, - nonAgg); + boolean nonAgg) + throws Exception { + List whereNode = + getWhereMeasure( + fields, + queryMetrics, + queryDimensions, + extendFields, + datasource, + scope, + schema, + nonAgg); dataSet.getMeasure().addAll(whereNode); - //getWhere(outputSet,fields,queryMetrics,queryDimensions,datasource,scope,schema); + // getWhere(outputSet,fields,queryMetrics,queryDimensions,datasource,scope,schema); } - public static void whereDimMetric(List fields, List queryMetrics, - List queryDimensions, DataSource datasource, SemanticSchema schema, Set dimensions, + public static void whereDimMetric( + List fields, + List queryMetrics, + List queryDimensions, + DataSource datasource, + SemanticSchema schema, + Set dimensions, Set metrics) { for (String field : fields) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) { @@ -262,42 +341,59 @@ public class SourceRender extends Renderer { } } - private static void addField(String field, String oriField, DataSource datasource, SemanticSchema schema, + private static void addField( + String field, + String oriField, + DataSource datasource, + SemanticSchema schema, Set dimensions, Set metrics) { - Optional dimension = datasource.getDimensions().stream() - .filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); + Optional dimension = + datasource.getDimensions().stream() + .filter(d -> d.getName().equalsIgnoreCase(field)) + .findFirst(); if (dimension.isPresent()) { dimensions.add(oriField); return; } - Optional identify = datasource.getIdentifiers().stream() - .filter(i -> i.getName().equalsIgnoreCase(field)).findFirst(); + Optional identify = + datasource.getIdentifiers().stream() + .filter(i -> i.getName().equalsIgnoreCase(field)) + .findFirst(); if (identify.isPresent()) { dimensions.add(oriField); return; } if (schema.getDimension().containsKey(datasource.getName())) { - Optional dataSourceDim = schema.getDimension().get(datasource.getName()).stream() - .filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); + Optional dataSourceDim = + schema.getDimension().get(datasource.getName()).stream() + .filter(d -> d.getName().equalsIgnoreCase(field)) + .findFirst(); if (dataSourceDim.isPresent()) { dimensions.add(oriField); return; } } - Optional metric = datasource.getMeasures() - .stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst(); + Optional metric = + datasource.getMeasures().stream() + .filter(m -> m.getName().equalsIgnoreCase(field)) + .findFirst(); if (metric.isPresent()) { metrics.add(oriField); return; } - Optional datasourceMetric = schema.getMetrics() - .stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst(); + Optional datasourceMetric = + schema.getMetrics().stream() + .filter(m -> m.getName().equalsIgnoreCase(field)) + .findFirst(); if (datasourceMetric.isPresent()) { - Set measures = datasourceMetric.get().getMetricTypeParams().getMeasures().stream() - .map(m -> m.getName()).collect( - Collectors.toSet()); - if (datasource.getMeasures().stream().map(m -> m.getName()).collect(Collectors.toSet()) + Set measures = + datasourceMetric.get().getMetricTypeParams().getMeasures().stream() + .map(m -> m.getName()) + .collect(Collectors.toSet()); + if (datasource.getMeasures().stream() + .map(m -> m.getName()) + .collect(Collectors.toSet()) .containsAll(measures)) { metrics.add(oriField); return; @@ -306,19 +402,25 @@ public class SourceRender extends Renderer { } public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) { - Optional dimension = datasource.getDimensions().stream() - .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); + Optional dimension = + datasource.getDimensions().stream() + .filter(d -> d.getName().equalsIgnoreCase(name)) + .findFirst(); if (dimension.isPresent()) { return true; } - Optional identify = datasource.getIdentifiers().stream() - .filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); + Optional identify = + datasource.getIdentifiers().stream() + .filter(i -> i.getName().equalsIgnoreCase(name)) + .findFirst(); if (identify.isPresent()) { return true; } if (schema.getDimension().containsKey(datasource.getName())) { - Optional dataSourceDim = schema.getDimension().get(datasource.getName()).stream() - .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); + Optional dataSourceDim = + schema.getDimension().get(datasource.getName()).stream() + .filter(d -> d.getName().equalsIgnoreCase(name)) + .findFirst(); if (dataSourceDim.isPresent()) { return true; } @@ -328,12 +430,30 @@ public class SourceRender extends Renderer { private static void addTimeDimension(DataSource dataSource, List queryDimension) { if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { - Optional startTimeOp = dataSource.getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) - .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START)).findFirst(); - Optional endTimeOp = dataSource.getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) - .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)).findFirst(); + Optional startTimeOp = + dataSource.getDimensions().stream() + .filter( + d -> + Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase( + d.getType())) + .filter( + d -> + d.getName() + .startsWith( + Constants.MATERIALIZATION_ZIPPER_START)) + .findFirst(); + Optional endTimeOp = + dataSource.getDimensions().stream() + .filter( + d -> + Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase( + d.getType())) + .filter( + d -> + d.getName() + .startsWith( + Constants.MATERIALIZATION_ZIPPER_END)) + .findFirst(); if (startTimeOp.isPresent() && !queryDimension.contains(startTimeOp.get().getName())) { queryDimension.add(startTimeOp.get().getName()); } @@ -341,20 +461,31 @@ public class SourceRender extends Renderer { queryDimension.add(endTimeOp.get().getName()); } } else { - Optional timeOp = dataSource.getDimensions().stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst(); + Optional timeOp = + dataSource.getDimensions().stream() + .filter( + d -> + Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase( + d.getType())) + .findFirst(); if (timeOp.isPresent() && !queryDimension.contains(timeOp.get().getName())) { queryDimension.add(timeOp.get().getName()); } } } - public void render(MetricQueryParam metricQueryParam, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + public void render( + MetricQueryParam metricQueryParam, + List dataSources, + SqlValidatorScope scope, + SemanticSchema schema, + boolean nonAgg) + throws Exception { String queryWhere = metricQueryParam.getWhere(); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); - EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); if (queryWhere != null && !queryWhere.isEmpty()) { SqlNode sqlNode = SemanticNode.parse(queryWhere, scope, engineType); FilterNode.getFilterField(sqlNode, whereFields); @@ -362,14 +493,21 @@ public class SourceRender extends Renderer { } if (dataSources.size() == 1) { DataSource dataSource = dataSources.get(0); - super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(), - metricQueryParam.getDimensions(), - metricQueryParam.getWhere(), dataSource, scope, schema, nonAgg); + super.tableView = + renderOne( + "", + fieldWhere, + metricQueryParam.getMetrics(), + metricQueryParam.getDimensions(), + metricQueryParam.getWhere(), + dataSource, + scope, + schema, + nonAgg); return; } JoinRender joinRender = new JoinRender(); joinRender.render(metricQueryParam, dataSources, scope, schema, nonAgg); super.tableView = joinRender.getTableView(); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java index cad030f2f..1739b6b0c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java @@ -13,30 +13,28 @@ import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; -/** - * supplement the QueryStatement when query with custom aggregation method - */ +/** supplement the QueryStatement when query with custom aggregation method */ @Component("CalculateAggConverter") @Slf4j public class CalculateAggConverter implements QueryConverter { - public interface EngineSql { String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql); } - public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement, - EngineType engineTypeEnum, String version) + public DataSetQueryParam generateSqlCommend( + QueryStatement queryStatement, EngineType engineTypeEnum, String version) throws Exception { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); QueryParam queryParam = queryStatement.getQueryParam(); @@ -55,16 +53,24 @@ public class CalculateAggConverter implements QueryConverter { metricTable.setWhere(where); metricTable.setAggOption(AggOption.AGGREGATION); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); - String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryParam), - metricTableName, - sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam), - sqlGenerateUtils.getLimit(queryParam)); + String sql = + String.format( + "select %s from %s %s %s %s", + sqlGenerateUtils.getSelect(queryParam), + metricTableName, + sqlGenerateUtils.getGroupBy(queryParam), + sqlGenerateUtils.getOrderBy(queryParam), + sqlGenerateUtils.getLimit(queryParam)); if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) { sqlCommand.setSupportWith(false); - sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryParam), - metricTableName, - sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam), - sqlGenerateUtils.getLimit(queryParam)); + sql = + String.format( + "select %s from %s t0 %s %s %s", + sqlGenerateUtils.getSelect(queryParam), + metricTableName, + sqlGenerateUtils.getGroupBy(queryParam), + sqlGenerateUtils.getOrderBy(queryParam), + sqlGenerateUtils.getLimit(queryParam)); } sqlCommand.setSql(sql); return sqlCommand; @@ -101,27 +107,31 @@ public class CalculateAggConverter implements QueryConverter { @Override public void convert(QueryStatement queryStatement) throws Exception { Database database = queryStatement.getSemanticModel().getDatabase(); - DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement, - EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); + DataSetQueryParam dataSetQueryParam = + generateSqlCommend( + queryStatement, + EngineType.fromString(database.getType().toUpperCase()), + database.getVersion()); queryStatement.setDataSetQueryParam(dataSetQueryParam); } - /** - * Ratio - */ - + /** Ratio */ public boolean isRatioAccept(QueryParam queryParam) { - Long ratioFuncNum = queryParam.getAggregators().stream() - .filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc() - .equals(AggOperatorEnum.RATIO_OVER))).count(); + Long ratioFuncNum = + queryParam.getAggregators().stream() + .filter( + f -> + (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) + || f.getFunc().equals(AggOperatorEnum.RATIO_OVER))) + .count(); if (ratioFuncNum > 0) { return true; } return false; } - public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum, - String version) + public DataSetQueryParam generateRatioSqlCommand( + QueryStatement queryStatement, EngineType engineTypeEnum, String version) throws Exception { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); QueryParam queryParam = queryStatement.getQueryParam(); @@ -151,9 +161,21 @@ public class CalculateAggConverter implements QueryConverter { sqlCommand.setSupportWith(false); } if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) { - sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), metricTableName); + sql = + new MysqlEngineSql() + .sql( + queryParam, + isOver, + sqlCommand.isSupportWith(), + metricTableName); } else { - sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), metricTableName); + sql = + new CkEngineSql() + .sql( + queryParam, + isOver, + sqlCommand.isSupportWith(), + metricTableName); } break; default: @@ -165,16 +187,27 @@ public class CalculateAggConverter implements QueryConverter { public class H2EngineSql implements EngineSql { public String getOverSelect(QueryParam queryParam, boolean isOver) { - String aggStr = queryParam.getAggregators().stream().map(f -> { - if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s", - f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), - f.getFunc().getOperator(), f.getColumn()); - } else { - return f.getColumn(); - } - }).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr + String aggStr = + queryParam.getAggregators().stream() + .map( + f -> { + if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) + || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { + return String.format( + "( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s", + f.getColumn(), + f.getColumn(), + f.getColumn(), + f.getColumn(), + f.getFunc().getOperator(), + f.getColumn()); + } else { + return f.getColumn(); + } + }) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(queryParam.getGroups()) + ? aggStr : String.join(",", queryParam.getGroups()) + "," + aggStr; } @@ -194,26 +227,48 @@ public class CalculateAggConverter implements QueryConverter { return ""; } - public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) { + public String getJoinOn( + QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) { String timeDim = getTimeDim(queryParam); String timeSpan = getTimeSpan(queryParam, isOver, true); - String aggStr = queryParam.getAggregators().stream().map(f -> { - if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) { - return String.format( - "%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ", - aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim); - } - if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) { - return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ", - getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim, aliasRight + timeDim); - } - return String.format("%s = TIMESTAMPADD(%s,%s) ", - aliasLeft + timeDim, timeSpan, aliasRight + timeDim); - } else { - return f.getColumn(); - } - }).collect(Collectors.joining(" and ")); + String aggStr = + queryParam.getAggregators().stream() + .map( + f -> { + if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) + || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { + if (queryParam + .getDateInfo() + .getPeriod() + .equals(Constants.MONTH)) { + return String.format( + "%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ", + aliasRight + timeDim, + aliasLeft + timeDim, + timeSpan, + aliasRight + timeDim); + } + if (queryParam + .getDateInfo() + .getPeriod() + .equals(Constants.WEEK) + && isOver) { + return String.format( + " DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ", + getTimeSpan(queryParam, isOver, false), + aliasLeft + timeDim, + aliasRight + timeDim); + } + return String.format( + "%s = TIMESTAMPADD(%s,%s) ", + aliasLeft + timeDim, + timeSpan, + aliasRight + timeDim); + } else { + return f.getColumn(); + } + }) + .collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); for (String group : queryParam.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { @@ -221,43 +276,71 @@ public class CalculateAggConverter implements QueryConverter { } groups.add(aliasLeft + group + " = " + aliasRight + group); } - return CollectionUtils.isEmpty(groups) ? aggStr + return CollectionUtils.isEmpty(groups) + ? aggStr : String.join(" and ", groups) + " and " + aggStr + " "; } @Override public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) { - String sql = String.format( - "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), - getAllJoinSelect(queryParam, "t1."), metricSql, metricSql, - getJoinOn(queryParam, isOver, "t0.", "t1."), - getOrderBy(queryParam), getLimit(queryParam)); + String sql = + String.format( + "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", + getOverSelect(queryParam, isOver), + getAllSelect(queryParam, "t0."), + getAllJoinSelect(queryParam, "t1."), + metricSql, + metricSql, + getJoinOn(queryParam, isOver, "t0.", "t1."), + getOrderBy(queryParam), + getLimit(queryParam)); return sql; } } public class CkEngineSql extends MysqlEngineSql { - public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) { + public String getJoinOn( + QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) { String timeDim = getTimeDim(queryParam); String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true); - String aggStr = queryParam.getAggregators().stream().map(f -> { - if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) { - return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ", - aliasLeft + timeDim, aliasRight + timeDim, timeSpan); - } - if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) { - return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s", - aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), aliasRight + timeDim); - } - return String.format("%s = date_add(%s,%s) ", - aliasLeft + timeDim, aliasRight + timeDim, timeSpan); - } else { - return f.getColumn(); - } - }).collect(Collectors.joining(" and ")); + String aggStr = + queryParam.getAggregators().stream() + .map( + f -> { + if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) + || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { + if (queryParam + .getDateInfo() + .getPeriod() + .equals(Constants.MONTH)) { + return String.format( + "toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ", + aliasLeft + timeDim, + aliasRight + timeDim, + timeSpan); + } + if (queryParam + .getDateInfo() + .getPeriod() + .equals(Constants.WEEK) + && isOver) { + return String.format( + "toMonday(date_add(%s ,INTERVAL %s) ) = %s", + aliasLeft + timeDim, + getTimeSpan(queryParam, isOver, false), + aliasRight + timeDim); + } + return String.format( + "%s = date_add(%s,%s) ", + aliasLeft + timeDim, + aliasRight + timeDim, + timeSpan); + } else { + return f.getColumn(); + } + }) + .collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); for (String group : queryParam.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { @@ -265,7 +348,8 @@ public class CalculateAggConverter implements QueryConverter { } groups.add(aliasLeft + group + " = " + aliasRight + group); } - return CollectionUtils.isEmpty(groups) ? aggStr + return CollectionUtils.isEmpty(groups) + ? aggStr : String.join(" and ", groups) + " and " + aggStr + " "; } @@ -274,18 +358,26 @@ public class CalculateAggConverter implements QueryConverter { if (!asWith) { return String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), - getAllJoinSelect(queryParam, "t1."), metricSql, metricSql, + getOverSelect(queryParam, isOver), + getAllSelect(queryParam, "t0."), + getAllJoinSelect(queryParam, "t1."), + metricSql, + metricSql, getJoinOn(queryParam, isOver, "t0.", "t1."), - getOrderBy(queryParam), getLimit(queryParam)); + getOrderBy(queryParam), + getLimit(queryParam)); } return String.format( ",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s " + "from t0 left join t1 on %s ) metric_tb_src %s %s ", - metricSql, metricSql, getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), + metricSql, + metricSql, + getOverSelect(queryParam, isOver), + getAllSelect(queryParam, "t0."), getAllJoinSelect(queryParam, "t1."), getJoinOn(queryParam, isOver, "t0.", "t1."), - getOrderBy(queryParam), getLimit(queryParam)); + getOrderBy(queryParam), + getLimit(queryParam)); } } @@ -308,39 +400,72 @@ public class CalculateAggConverter implements QueryConverter { } public String getOverSelect(QueryParam queryParam, boolean isOver) { - String aggStr = queryParam.getAggregators().stream().map(f -> { - if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - return String.format( - "if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s", - f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), - f.getColumn(), f.getFunc().getOperator(), f.getColumn()); - } else { - return f.getColumn(); - } - }).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr + String aggStr = + queryParam.getAggregators().stream() + .map( + f -> { + if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) + || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { + return String.format( + "if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s", + f.getColumn(), + f.getColumn(), + f.getColumn(), + f.getColumn(), + f.getColumn(), + f.getFunc().getOperator(), + f.getColumn()); + } else { + return f.getColumn(); + } + }) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(queryParam.getGroups()) + ? aggStr : String.join(",", queryParam.getGroups()) + "," + aggStr; } - public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) { + public String getJoinOn( + QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) { String timeDim = getTimeDim(queryParam); String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true); - String aggStr = queryParam.getAggregators().stream().map(f -> { - if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) { - return String.format("%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ", - aliasLeft + timeDim, aliasRight + timeDim, timeSpan); - } - if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) { - return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s", - aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), aliasRight + timeDim); - } - return String.format("%s = date_add(%s,%s) ", - aliasLeft + timeDim, aliasRight + timeDim, timeSpan); - } else { - return f.getColumn(); - } - }).collect(Collectors.joining(" and ")); + String aggStr = + queryParam.getAggregators().stream() + .map( + f -> { + if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) + || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { + if (queryParam + .getDateInfo() + .getPeriod() + .equals(Constants.MONTH)) { + return String.format( + "%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ", + aliasLeft + timeDim, + aliasRight + timeDim, + timeSpan); + } + if (queryParam + .getDateInfo() + .getPeriod() + .equals(Constants.WEEK) + && isOver) { + return String.format( + "to_monday(date_add(%s ,INTERVAL %s) ) = %s", + aliasLeft + timeDim, + getTimeSpan(queryParam, isOver, false), + aliasRight + timeDim); + } + return String.format( + "%s = date_add(%s,%s) ", + aliasLeft + timeDim, + aliasRight + timeDim, + timeSpan); + } else { + return f.getColumn(); + } + }) + .collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); for (String group : queryParam.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { @@ -348,39 +473,49 @@ public class CalculateAggConverter implements QueryConverter { } groups.add(aliasLeft + group + " = " + aliasRight + group); } - return CollectionUtils.isEmpty(groups) ? aggStr + return CollectionUtils.isEmpty(groups) + ? aggStr : String.join(" and ", groups) + " and " + aggStr + " "; } @Override public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) { - String sql = String.format( - "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), - getAllJoinSelect(queryParam, "t1."), metricSql, metricSql, - getJoinOn(queryParam, isOver, "t0.", "t1."), - getOrderBy(queryParam), getLimit(queryParam)); + String sql = + String.format( + "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", + getOverSelect(queryParam, isOver), + getAllSelect(queryParam, "t0."), + getAllJoinSelect(queryParam, "t1."), + metricSql, + metricSql, + getJoinOn(queryParam, isOver, "t0.", "t1."), + getOrderBy(queryParam), + getLimit(queryParam)); return sql; } } private String getAllJoinSelect(QueryParam queryParam, String alias) { - String aggStr = queryParam.getAggregators().stream() - .map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") - + "_roll") - .collect(Collectors.joining(",")); + String aggStr = + queryParam.getAggregators().stream() + .map( + f -> + getSelectField(f, alias) + + " as " + + getSelectField(f, "") + + "_roll") + .collect(Collectors.joining(",")); List groups = new ArrayList<>(); for (String group : queryParam.getGroups()) { groups.add(alias + group + " as " + group + "_roll"); } - return CollectionUtils.isEmpty(groups) ? aggStr - : String.join(",", groups) + "," + aggStr; - + return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr; } private String getGroupDimWithOutTime(QueryParam queryParam) { String timeDim = getTimeDim(queryParam); - return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim)) + return queryParam.getGroups().stream() + .filter(f -> !f.equalsIgnoreCase(timeDim)) .collect(Collectors.joining(",")); } @@ -397,15 +532,19 @@ public class CalculateAggConverter implements QueryConverter { } private String getAllSelect(QueryParam queryParam, String alias) { - String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias)) - .collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr + String aggStr = + queryParam.getAggregators().stream() + .map(f -> getSelectField(f, alias)) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(queryParam.getGroups()) + ? aggStr : alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr; } private String getSelectField(final Aggregator agg, String alias) { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - if (agg.getFunc().equals(AggOperatorEnum.RATIO_OVER) || agg.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { + if (agg.getFunc().equals(AggOperatorEnum.RATIO_OVER) + || agg.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return alias + agg.getColumn(); } return sqlGenerateUtils.getSelectField(agg); @@ -423,22 +562,27 @@ public class CalculateAggConverter implements QueryConverter { } private boolean isOverRatio(QueryParam queryParam) { - Long overCt = queryParam.getAggregators().stream() - .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); + Long overCt = + queryParam.getAggregators().stream() + .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)) + .count(); return overCt > 0; } private void check(QueryParam queryParam) throws Exception { - Long ratioOverNum = queryParam.getAggregators().stream() - .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); - Long ratioRollNum = queryParam.getAggregators().stream() - .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count(); + Long ratioOverNum = + queryParam.getAggregators().stream() + .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)) + .count(); + Long ratioRollNum = + queryParam.getAggregators().stream() + .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) + .count(); if (ratioOverNum > 0 && ratioRollNum > 0) { throw new Exception("not support over ratio and roll ratio together "); } if (getTimeDim(queryParam).isEmpty()) { throw new Exception("miss time filter"); } - } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java index 26307786b..28f0f8572 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.headless.core.translator.converter; import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.MetricTable; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.StringValue; @@ -16,6 +16,7 @@ import net.sf.jsqlparser.schema.Column; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -33,21 +34,23 @@ public class DefaultDimValueConverter implements QueryConverter { @Override public void convert(QueryStatement queryStatement) { - List dimensions = queryStatement.getSemanticModel().getDimensions().stream() - .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) - .collect(Collectors.toList()); + List dimensions = + queryStatement.getSemanticModel().getDimensions().stream() + .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dimensions)) { return; } String sql = queryStatement.getDataSetQueryParam().getSql(); - List whereFields = SqlSelectHelper.getWhereFields(sql) - .stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) - .collect(Collectors.toList()); + List whereFields = + SqlSelectHelper.getWhereFields(sql).stream() + .filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) + .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(whereFields)) { return; } - MetricTable metricTable = queryStatement.getDataSetQueryParam() - .getTables().stream().findFirst().orElse(null); + MetricTable metricTable = + queryStatement.getDataSetQueryParam().getTables().stream().findFirst().orElse(null); List expressions = Lists.newArrayList(); for (Dimension dimension : dimensions) { ExpressionList expressionList = new ExpressionList(); @@ -65,5 +68,4 @@ public class DefaultDimValueConverter implements QueryConverter { sql = SqlAddHelper.addWhere(sql, expressions); queryStatement.getDataSetQueryParam().setSql(sql); } - -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java index e6a6fdef9..731df5cbd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java @@ -3,21 +3,20 @@ package com.tencent.supersonic.headless.core.translator.converter; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.QueryParam; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -/** - * QueryConverter default implement - */ +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** QueryConverter default implement */ @Component("ParserDefaultConverter") @Slf4j public class ParserDefaultConverter implements QueryConverter { @@ -27,7 +26,8 @@ public class ParserDefaultConverter implements QueryConverter { if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) { return false; } - CalculateAggConverter calculateConverterAgg = ContextUtils.getBean(CalculateAggConverter.class); + CalculateAggConverter calculateConverterAgg = + ContextUtils.getBean(CalculateAggConverter.class); return !calculateConverterAgg.accept(queryStatement); } @@ -36,12 +36,14 @@ public class ParserDefaultConverter implements QueryConverter { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); QueryParam queryParam = queryStatement.getQueryParam(); MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam(); - MetricQueryParam metricReq = generateSqlCommand(queryStatement.getQueryParam(), queryStatement); + MetricQueryParam metricReq = + generateSqlCommand(queryStatement.getQueryParam(), queryStatement); queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryParam, null)); BeanUtils.copyProperties(metricReq, metricQueryParam); } - public MetricQueryParam generateSqlCommand(QueryParam queryParam, QueryStatement queryStatement) { + public MetricQueryParam generateSqlCommand( + QueryParam queryParam, QueryStatement queryStatement) { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); MetricQueryParam metricQueryParam = new MetricQueryParam(); metricQueryParam.setMetrics(queryParam.getMetrics()); @@ -50,21 +52,24 @@ public class ParserDefaultConverter implements QueryConverter { log.info("in generateSqlCommend, complete where:{}", where); metricQueryParam.setWhere(where); - metricQueryParam.setOrder(queryParam.getOrders().stream() - .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList())); + metricQueryParam.setOrder( + queryParam.getOrders().stream() + .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())) + .collect(Collectors.toList())); metricQueryParam.setLimit(queryParam.getLimit()); // support detail query - if (queryParam.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) { + if (queryParam.getQueryType().isNativeAggQuery() + && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) { Map modelMap = queryStatement.getSemanticModel().getModelMap(); for (Long modelId : modelMap.keySet()) { String modelBizName = modelMap.get(modelId).getName(); - String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName); + String internalMetricName = + sqlGenerateUtils.generateInternalMetricName(modelBizName); metricQueryParam.getMetrics().add(internalMetricName); } } return metricQueryParam; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java index abb4a2b15..cb2b46bd1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java @@ -2,13 +2,10 @@ package com.tencent.supersonic.headless.core.translator.converter; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -/** - * to supplement,translate the request Body - */ +/** to supplement,translate the request Body */ public interface QueryConverter { boolean accept(QueryStatement queryStatement); void convert(QueryStatement queryStatement) throws Exception; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java index 39ccbadd7..e6a05d302 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.translator.converter; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -33,17 +33,21 @@ public class SqlVariableParseConverter implements QueryConverter { return; } for (ModelResp modelResp : modelResps) { - if (ModelDefineType.SQL_QUERY.getName() + if (ModelDefineType.SQL_QUERY + .getName() .equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) { - String sqlParsed = SqlVariableParseUtils.parse( - modelResp.getModelDetail().getSqlQuery(), - modelResp.getModelDetail().getSqlVariables(), - queryStatement.getQueryParam().getParams() - ); - DataSource dataSource = queryStatement.getSemanticModel() - .getDatasourceMap().get(modelResp.getBizName()); + String sqlParsed = + SqlVariableParseUtils.parse( + modelResp.getModelDetail().getSqlQuery(), + modelResp.getModelDetail().getSqlVariables(), + queryStatement.getQueryParam().getParams()); + DataSource dataSource = + queryStatement + .getSemanticModel() + .getDatasourceMap() + .get(modelResp.getBizName()); dataSource.setSqlQuery(sqlParsed); } } } -} \ No newline at end of file +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index 38cecd467..2e37a3c6c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -2,23 +2,21 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.core.cache.QueryCache; -import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.executor.QueryAccelerator; +import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.core.translator.QueryOptimizer; import com.tencent.supersonic.headless.core.translator.QueryParser; import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; -import com.tencent.supersonic.headless.core.translator.QueryOptimizer; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.support.SpringFactoriesLoader; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.core.io.support.SpringFactoriesLoader; - -/** - * QueryConverter QueryOptimizer QueryExecutor object factory - */ +/** QueryConverter QueryOptimizer QueryExecutor object factory */ @Slf4j public class ComponentFactory { @@ -88,17 +86,20 @@ public class ComponentFactory { List queryOptimizerList = new ArrayList<>(); init(QueryOptimizer.class, queryOptimizerList); if (!queryOptimizerList.isEmpty()) { - queryOptimizerList.stream().forEach(q -> addQueryOptimizer(q.getClass().getSimpleName(), q)); + queryOptimizerList.stream() + .forEach(q -> addQueryOptimizer(q.getClass().getSimpleName(), q)); } } private static void initQueryExecutors() { - //queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class)); + // queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", + // JdbcExecutor.class)); init(QueryExecutor.class, queryExecutors); } private static void initQueryAccelerators() { - //queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class)); + // queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", + // JdbcExecutor.class)); init(QueryAccelerator.class, queryAccelerators); } @@ -119,14 +120,15 @@ public class ComponentFactory { } private static List init(Class factoryType, List list) { - list.addAll(SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader())); + list.addAll( + SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader())); return list; } private static T init(Class factoryType) { - return SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader()).get(0); + return SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader()) + .get(0); } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java index 6ab8c3c2c..44be9f2b4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java @@ -5,19 +5,21 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import org.apache.commons.lang3.StringUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -/** - * transform query results to return the users - */ +/** transform query results to return the users */ public class DataTransformUtils { - public static List> transform(List> originalData, String metric, - List groups, DateConf dateConf) { + public static List> transform( + List> originalData, + String metric, + List groups, + DateConf dateConf) { List dateList = dateConf.getDateList(); List> transposedData = new ArrayList<>(); for (Map originalRow : originalData) { @@ -27,12 +29,14 @@ public class DataTransformUtils { transposedRow.put(key, originalRow.get(key)); } } - transposedRow.put(String.valueOf(originalRow.get(getTimeDimension(dateConf))), + transposedRow.put( + String.valueOf(originalRow.get(getTimeDimension(dateConf))), originalRow.get(metric)); transposedData.add(transposedRow); } - Map>> dataMerge = transposedData.stream() - .collect(Collectors.groupingBy(row -> getRowKey(row, groups))); + Map>> dataMerge = + transposedData.stream() + .collect(Collectors.groupingBy(row -> getRowKey(row, groups))); List> resultData = Lists.newArrayList(); for (List> data : dataMerge.values()) { Map rowData = new HashMap<>(); @@ -70,5 +74,4 @@ public class DataTransformUtils { return TimeDimensionEnum.DAY.getName(); } } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java index 7a11deed9..11ef81875 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.core.utils; +import javax.sql.DataSource; + import com.alibaba.druid.util.StringUtils; import com.tencent.supersonic.common.util.MD5Util; import com.tencent.supersonic.headless.api.pojo.enums.DataType; @@ -7,7 +9,7 @@ import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import javax.sql.DataSource; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -15,6 +17,7 @@ import java.sql.SQLException; import java.util.HashSet; import java.util.Set; import java.util.regex.Matcher; + import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; import static com.tencent.supersonic.common.pojo.Constants.COLON; import static com.tencent.supersonic.common.pojo.Constants.DOUBLE_SLASH; @@ -24,14 +27,11 @@ import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR; import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE; import static com.tencent.supersonic.common.pojo.Constants.SPACE; -/** - * tools functions about jdbc - */ +/** tools functions about jdbc */ @Slf4j public class JdbcDataSourceUtils { - @Getter - private static Set releaseSourceSet = new HashSet(); + @Getter private static Set releaseSourceSet = new HashSet(); private JdbcDataSource jdbcDataSource; public JdbcDataSourceUtils(JdbcDataSource jdbcDataSource) { @@ -46,8 +46,9 @@ public class JdbcDataSourceUtils { log.error(e.toString(), e); return false; } - try (Connection con = DriverManager.getConnection(database.getUrl(), - database.getUsername(), database.passwordDecrypt())) { + try (Connection con = + DriverManager.getConnection( + database.getUrl(), database.getUsername(), database.passwordDecrypt())) { return con != null; } catch (SQLException e) { log.error(e.toString(), e); @@ -115,7 +116,8 @@ public class JdbcDataSourceUtils { log.error("e", e); } - if (!StringUtils.isEmpty(className) && !className.contains("com.sun.proxy") + if (!StringUtils.isEmpty(className) + && !className.contains("com.sun.proxy") && !className.contains("net.sf.cglib.proxy")) { return className; } @@ -127,7 +129,12 @@ public class JdbcDataSourceUtils { throw new RuntimeException("Not supported data type: jdbcUrl=" + jdbcUrl); } - public static String getKey(String name, String jdbcUrl, String username, String password, String version, + public static String getKey( + String name, + String jdbcUrl, + String username, + String password, + String version, boolean isExt) { StringBuilder sb = new StringBuilder(); @@ -158,8 +165,10 @@ public class JdbcDataSourceUtils { return dataSource.getConnection(); } catch (Exception e) { log.error("Get connection error, jdbcUrl:{}, e:{}", database.getUrl(), e); - throw new RuntimeException("Get connection error, jdbcUrl:" + database.getUrl() - + " you can try again later or reset datasource"); + throw new RuntimeException( + "Get connection error, jdbcUrl:" + + database.getUrl() + + " you can try again later or reset datasource"); } } return conn; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDuckDbUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDuckDbUtils.java index b9fb394d0..89eb30fd9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDuckDbUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDuckDbUtils.java @@ -1,60 +1,64 @@ package com.tencent.supersonic.headless.core.utils; - import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.DuckDbSource; + import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -/** - * tools functions to duckDb query - */ +/** tools functions to duckDb query */ public class JdbcDuckDbUtils { - public static void attachMysql(DuckDbSource duckDbSource, String host, Integer port, String user, + public static void attachMysql( + DuckDbSource duckDbSource, + String host, + Integer port, + String user, String password, String database) throws Exception { try { duckDbSource.execute("INSTALL mysql"); duckDbSource.execute("load mysql"); - String attachSql = "ATTACH 'host=%s port=%s user=%s password=%s database=%s' AS mysqldb (TYPE mysql);"; - duckDbSource.execute(String.format(attachSql, - host, - port, - user, - password, - database - )); + String attachSql = + "ATTACH 'host=%s port=%s user=%s password=%s database=%s' AS mysqldb (TYPE mysql);"; + duckDbSource.execute(String.format(attachSql, host, port, user, password, database)); duckDbSource.execute("SET mysql_experimental_filter_pushdown = true;"); } catch (Exception e) { throw e; } } - public static List getParquetColumns(DuckDbSource duckDbSource, String parquetPath) throws Exception { + public static List getParquetColumns(DuckDbSource duckDbSource, String parquetPath) + throws Exception { SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); - duckDbSource.query(String.format("SELECT distinct name FROM parquet_schema('%s')", parquetPath), + duckDbSource.query( + String.format("SELECT distinct name FROM parquet_schema('%s')", parquetPath), queryResultWithColumns); if (!queryResultWithColumns.getResultList().isEmpty()) { return queryResultWithColumns.getResultList().stream() .filter(l -> l.containsKey("name") && Objects.nonNull(l.get("name"))) - .map(l -> (String) l.get("name")).collect(Collectors.toList()); + .map(l -> (String) l.get("name")) + .collect(Collectors.toList()); } return new ArrayList<>(); } - public static List getParquetPartition(DuckDbSource duckDbSource, String parquetPath, String partitionName) - throws Exception { + public static List getParquetPartition( + DuckDbSource duckDbSource, String parquetPath, String partitionName) throws Exception { SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); - duckDbSource.query(String.format("SELECT distinct %s as partition FROM read_parquet('%s')", partitionName, - parquetPath), queryResultWithColumns); + duckDbSource.query( + String.format( + "SELECT distinct %s as partition FROM read_parquet('%s')", + partitionName, parquetPath), + queryResultWithColumns); if (!queryResultWithColumns.getResultList().isEmpty()) { return queryResultWithColumns.getResultList().stream() .filter(l -> l.containsKey("partition") && Objects.nonNull(l.get("partition"))) - .map(l -> (String) l.get("partition")).collect(Collectors.toList()); + .map(l -> (String) l.get("partition")) + .collect(Collectors.toList()); } return new ArrayList<>(); } @@ -64,9 +68,9 @@ public class JdbcDuckDbUtils { return true; } - public static boolean createView(DuckDbSource duckDbSource, String view, String sql) throws Exception { + public static boolean createView(DuckDbSource duckDbSource, String view, String sql) + throws Exception { duckDbSource.execute(String.format("CREATE OR REPLACE VIEW %s AS %s;", view, sql)); return true; } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java index c14db70c8..7ca9930f3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java @@ -9,9 +9,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -/** - * Schema Match Helper - */ +/** Schema Match Helper */ @Slf4j public class SchemaMatchHelper { @@ -20,22 +18,25 @@ public class SchemaMatchHelper { return; } - Set metricDimensionDetectWordSet = matches.stream() - .filter(SchemaMatchHelper::isMetricOrDimension) - .map(SchemaElementMatch::getDetectWord).collect(Collectors.toSet()); + Set metricDimensionDetectWordSet = + matches.stream() + .filter(SchemaMatchHelper::isMetricOrDimension) + .map(SchemaElementMatch::getDetectWord) + .collect(Collectors.toSet()); - matches.removeIf(elementMatch -> { - if (!isMetricOrDimension(elementMatch)) { - return false; - } - for (String detectWord : metricDimensionDetectWordSet) { - if (detectWord.startsWith(elementMatch.getDetectWord()) - && detectWord.length() > elementMatch.getDetectWord().length()) { - return true; - } - } - return false; - }); + matches.removeIf( + elementMatch -> { + if (!isMetricOrDimension(elementMatch)) { + return false; + } + for (String detectWord : metricDimensionDetectWordSet) { + if (detectWord.startsWith(elementMatch.getDetectWord()) + && detectWord.length() > elementMatch.getDetectWord().length()) { + return true; + } + } + return false; + }); } private static boolean isMetricOrDimension(SchemaElementMatch elementMatch) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java index c33886601..005ee2fbd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java @@ -2,32 +2,34 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlDialect; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; import org.apache.calcite.avatica.util.Casing; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlDialect.Context; import org.apache.calcite.sql.SqlDialect.DatabaseProduct; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; public class SqlDialectFactory { - public static final Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT - .withDatabaseProduct(DatabaseProduct.BIG_QUERY) - .withLiteralQuoteString("'") - .withLiteralEscapedQuoteString("''") - .withIdentifierQuoteString("`") - .withUnquotedCasing(Casing.UNCHANGED) - .withQuotedCasing(Casing.UNCHANGED) - .withCaseSensitive(false); - public static final Context POSTGRESQL_CONTEXT = SqlDialect.EMPTY_CONTEXT - .withDatabaseProduct(DatabaseProduct.BIG_QUERY) - .withLiteralQuoteString("'") - .withLiteralEscapedQuoteString("''") - .withUnquotedCasing(Casing.UNCHANGED) - .withQuotedCasing(Casing.UNCHANGED) - .withCaseSensitive(false); + public static final Context DEFAULT_CONTEXT = + SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(DatabaseProduct.BIG_QUERY) + .withLiteralQuoteString("'") + .withLiteralEscapedQuoteString("''") + .withIdentifierQuoteString("`") + .withUnquotedCasing(Casing.UNCHANGED) + .withQuotedCasing(Casing.UNCHANGED) + .withCaseSensitive(false); + public static final Context POSTGRESQL_CONTEXT = + SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(DatabaseProduct.BIG_QUERY) + .withLiteralQuoteString("'") + .withLiteralEscapedQuoteString("''") + .withUnquotedCasing(Casing.UNCHANGED) + .withQuotedCasing(Casing.UNCHANGED) + .withCaseSensitive(false); private static Map sqlDialectMap; static { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index c10490c5c..e17ff2ce8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -46,11 +46,7 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH; import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; import static com.tencent.supersonic.common.pojo.Constants.WEEK; -; - -/** - * tools functions to analyze queryStructReq - */ +/** tools functions to analyze queryStructReq */ @Component @Slf4j public class SqlGenerateUtils { @@ -61,8 +57,10 @@ public class SqlGenerateUtils { private final ExecutorConfig executorConfig; - public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils, - DateModeUtils dateModeUtils, ExecutorConfig executorConfig) { + public SqlGenerateUtils( + SqlFilterUtils sqlFilterUtils, + DateModeUtils dateModeUtils, + ExecutorConfig executorConfig) { this.sqlFilterUtils = sqlFilterUtils; this.dateModeUtils = dateModeUtils; this.executorConfig = executorConfig; @@ -100,16 +98,22 @@ public class SqlGenerateUtils { } public String getSelect(QueryParam queryParam) { - String aggStr = queryParam.getAggregators().stream().map(this::getSelectField) - .collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr + String aggStr = + queryParam.getAggregators().stream() + .map(this::getSelectField) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(queryParam.getGroups()) + ? aggStr : String.join(",", queryParam.getGroups()) + "," + aggStr; } public String getSelect(QueryParam queryParam, Map deriveMetrics) { - String aggStr = queryParam.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics)) - .collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr + String aggStr = + queryParam.getAggregators().stream() + .map(a -> getSelectField(a, deriveMetrics)) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(queryParam.getGroups()) + ? aggStr : String.join(",", queryParam.getGroups()) + "," + aggStr; } @@ -120,9 +124,20 @@ public class SqlGenerateUtils { if (CollectionUtils.isEmpty(agg.getArgs())) { return agg.getFunc() + "( " + agg.getColumn() + " ) AS " + agg.getColumn() + " "; } - return agg.getFunc() + "( " + agg.getArgs().stream().map(arg -> - arg.equals(agg.getColumn()) ? arg : (StringUtils.isNumeric(arg) ? arg : ("'" + arg + "'")) - ).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " "; + return agg.getFunc() + + "( " + + agg.getArgs().stream() + .map( + arg -> + arg.equals(agg.getColumn()) + ? arg + : (StringUtils.isNumeric(arg) + ? arg + : ("'" + arg + "'"))) + .collect(Collectors.joining(",")) + + " ) AS " + + agg.getColumn() + + " "; } public String getSelectField(final Aggregator agg, Map deriveMetrics) { @@ -143,37 +158,51 @@ public class SqlGenerateUtils { if (CollectionUtils.isEmpty(queryParam.getOrders())) { return ""; } - return "order by " + queryParam.getOrders().stream() - .map(order -> " " + order.getColumn() + " " + order.getDirection() + " ") - .collect(Collectors.joining(",")); + return "order by " + + queryParam.getOrders().stream() + .map(order -> " " + order.getColumn() + " " + order.getDirection() + " ") + .collect(Collectors.joining(",")); } public String getOrderBy(QueryParam queryParam, Map deriveMetrics) { if (CollectionUtils.isEmpty(queryParam.getOrders())) { return ""; } - if (!queryParam.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) { + if (!queryParam.getOrders().stream() + .anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) { return getOrderBy(queryParam); } - return "order by " + queryParam.getOrders().stream() - .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn()) - : order.getColumn()) + " " + order.getDirection() + " ") - .collect(Collectors.joining(",")); + return "order by " + + queryParam.getOrders().stream() + .map( + order -> + " " + + (deriveMetrics.containsKey(order.getColumn()) + ? deriveMetrics.get(order.getColumn()) + : order.getColumn()) + + " " + + order.getDirection() + + " ") + .collect(Collectors.joining(",")); } public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) { - String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters()); + String whereClauseFromFilter = + sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters()); String whereFromDate = getDateWhereClause(queryParam.getDateInfo(), itemDateResp); return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate); } - private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter, - String whereFromDate) { - if (StringUtils.isNotEmpty(whereFromDate) && StringUtils.isNotEmpty(whereClauseFromFilter)) { + private String mergeDateWhereClause( + QueryParam queryParam, String whereClauseFromFilter, String whereFromDate) { + if (StringUtils.isNotEmpty(whereFromDate) + && StringUtils.isNotEmpty(whereClauseFromFilter)) { return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); - } else if (StringUtils.isEmpty(whereFromDate) && StringUtils.isNotEmpty(whereClauseFromFilter)) { + } else if (StringUtils.isEmpty(whereFromDate) + && StringUtils.isNotEmpty(whereClauseFromFilter)) { return whereClauseFromFilter; - } else if (StringUtils.isNotEmpty(whereFromDate) && StringUtils.isEmpty(whereClauseFromFilter)) { + } else if (StringUtils.isNotEmpty(whereFromDate) + && StringUtils.isEmpty(whereClauseFromFilter)) { return whereFromDate; } else if (Objects.isNull(whereFromDate) && StringUtils.isEmpty(whereClauseFromFilter)) { log.debug("the current date information is empty, enter the date initialization logic"); @@ -185,7 +214,7 @@ public class SqlGenerateUtils { public String getDateWhereClause(DateConf dateInfo, ItemDateResp dateDate) { if (Objects.isNull(dateDate) || StringUtils.isEmpty(dateDate.getStartDate()) - && StringUtils.isEmpty(dateDate.getEndDate())) { + && StringUtils.isEmpty(dateDate.getEndDate())) { if (dateInfo.getDateMode().equals(DateConf.DateMode.LIST)) { return dateModeUtils.listDateStr(dateInfo); } @@ -202,7 +231,8 @@ public class SqlGenerateUtils { return dateModeUtils.getDateWhereStr(dateInfo, dateDate); } - public Triple getBeginEndTime(QueryParam queryParam, ItemDateResp dataDate) { + public Triple getBeginEndTime( + QueryParam queryParam, ItemDateResp dataDate) { if (Objects.isNull(queryParam.getDateInfo())) { return Triple.of("", "", ""); } @@ -216,27 +246,38 @@ public class SqlGenerateUtils { case BETWEEN: return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate()); case LIST: - return Triple.of(dateInfo, Collections.min(dateConf.getDateList()), + return Triple.of( + dateInfo, + Collections.min(dateConf.getDateList()), Collections.max(dateConf.getDateList())); case RECENT: LocalDate dateMax = LocalDate.now().minusDays(1); LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1); if (Objects.isNull(dataDate)) { - return Triple.of(dateInfo, dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)), + return Triple.of( + dateInfo, + dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)), dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT))); } switch (dateConf.getPeriod()) { case DAY: - ImmutablePair dayInfo = dateModeUtils.recentDay(dataDate, dateConf); + ImmutablePair dayInfo = + dateModeUtils.recentDay(dataDate, dateConf); return Triple.of(dateInfo, dayInfo.left, dayInfo.right); case WEEK: - ImmutablePair weekInfo = dateModeUtils.recentWeek(dataDate, dateConf); + ImmutablePair weekInfo = + dateModeUtils.recentWeek(dataDate, dateConf); return Triple.of(dateInfo, weekInfo.left, weekInfo.right); case MONTH: - List> rets = dateModeUtils.recentMonth(dataDate, dateConf); - Optional minBegins = rets.stream().map(i -> i.left).sorted().findFirst(); - Optional maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder()) - .findFirst(); + List> rets = + dateModeUtils.recentMonth(dataDate, dateConf); + Optional minBegins = + rets.stream().map(i -> i.left).sorted().findFirst(); + Optional maxBegins = + rets.stream() + .map(i -> i.right) + .sorted(Comparator.reverseOrder()) + .findFirst(); if (minBegins.isPresent() && maxBegins.isPresent()) { return Triple.of(dateInfo, minBegins.get(), maxBegins.get()); } @@ -247,19 +288,19 @@ public class SqlGenerateUtils { break; default: break; - } return Triple.of("", "", ""); } public boolean isSupportWith(EngineType engineTypeEnum, String version) { - if (engineTypeEnum.equals(EngineType.MYSQL) && Objects.nonNull(version) && version.startsWith( - executorConfig.getMysqlLowVersion())) { + if (engineTypeEnum.equals(EngineType.MYSQL) + && Objects.nonNull(version) + && version.startsWith(executorConfig.getMysqlLowVersion())) { return false; } - if (engineTypeEnum.equals(EngineType.CLICKHOUSE) && Objects.nonNull(version) - && StringUtil.compareVersion(version, - executorConfig.getCkLowVersion()) < 0) { + if (engineTypeEnum.equals(EngineType.CLICKHOUSE) + && Objects.nonNull(version) + && StringUtil.compareVersion(version, executorConfig.getCkLowVersion()) < 0) { return false; } return true; @@ -269,30 +310,44 @@ public class SqlGenerateUtils { return modelBizName + UNDERLINE + executorConfig.getInternalMetricNameSuffix(); } - public String generateDerivedMetric(final List metricResps, final Set allFields, - final Map allMeasures, - final List dimensionResps, - final String expression, final MetricDefineType metricDefineType, - AggOption aggOption, - Set visitedMetric, - Set measures, - Set dimensions) { + public String generateDerivedMetric( + final List metricResps, + final Set allFields, + final Map allMeasures, + final List dimensionResps, + final String expression, + final MetricDefineType metricDefineType, + AggOption aggOption, + Set visitedMetric, + Set measures, + Set dimensions) { Set fields = SqlSelectHelper.getColumnFromExpr(expression); if (!CollectionUtils.isEmpty(fields)) { Map replace = new HashMap<>(); for (String field : fields) { switch (metricDefineType) { case METRIC: - Optional metricItem = metricResps.stream() - .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst(); + Optional metricItem = + metricResps.stream() + .filter(m -> m.getBizName().equalsIgnoreCase(field)) + .findFirst(); if (metricItem.isPresent()) { if (visitedMetric.contains(field)) { break; } - replace.put(field, - generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps, - getExpr(metricItem.get()), metricItem.get().getMetricDefineType(), - aggOption, visitedMetric, measures, dimensions)); + replace.put( + field, + generateDerivedMetric( + metricResps, + allFields, + allMeasures, + dimensionResps, + getExpr(metricItem.get()), + metricItem.get().getMetricDefineType(), + aggOption, + visitedMetric, + measures, + dimensions)); visitedMetric.add(field); } break; @@ -304,8 +359,10 @@ public class SqlGenerateUtils { break; case FIELD: if (allFields.contains(field)) { - Optional dimensionItem = dimensionResps.stream() - .filter(d -> d.getBizName().equals(field)).findFirst(); + Optional dimensionItem = + dimensionResps.stream() + .filter(d -> d.getBizName().equals(field)) + .findFirst(); if (dimensionItem.isPresent()) { dimensions.add(field); } else { @@ -315,7 +372,6 @@ public class SqlGenerateUtils { break; default: break; - } } if (!CollectionUtils.isEmpty(replace)) { @@ -329,12 +385,17 @@ public class SqlGenerateUtils { public String getExpr(Measure measure, AggOption aggOption) { if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) { - return AggOption.NATIVE.equals(aggOption) ? measure.getBizName() - : AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " " - + measure.getBizName() - + " ) "; + return AggOption.NATIVE.equals(aggOption) + ? measure.getBizName() + : AggOperatorEnum.COUNT.getOperator() + + " ( " + + AggOperatorEnum.DISTINCT + + " " + + measure.getBizName() + + " ) "; } - return AggOption.NATIVE.equals(aggOption) ? measure.getBizName() + return AggOption.NATIVE.equals(aggOption) + ? measure.getBizName() : measure.getAgg() + " ( " + measure.getBizName() + " ) "; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java index 9854fac9d..9385154ba 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.core.utils; +import javax.sql.DataSource; + import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.enums.DataType; @@ -13,7 +15,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import javax.sql.DataSource; import java.rmi.ServerException; import java.sql.Connection; import java.sql.ResultSet; @@ -30,18 +31,14 @@ import java.util.Map; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; -/** - * tools functions about sql query - */ +/** tools functions about sql query */ @Slf4j @Component public class SqlUtils { - @Getter - private Database database; + @Getter private Database database; - @Autowired - private JdbcDataSource jdbcDataSource; + @Autowired private JdbcDataSource jdbcDataSource; @Value("${s2.source.result-limit:1000000}") private int resultLimit; @@ -49,15 +46,11 @@ public class SqlUtils { @Value("${s2.source.enable-query-log:false}") private boolean isQueryLogEnable; - @Getter - private DataType dataTypeEnum; + @Getter private DataType dataTypeEnum; - @Getter - private JdbcDataSourceUtils jdbcDataSourceUtils; + @Getter private JdbcDataSourceUtils jdbcDataSourceUtils; - public SqlUtils() { - - } + public SqlUtils() {} public SqlUtils(Database database) { this.database = database; @@ -65,8 +58,7 @@ public class SqlUtils { } public SqlUtils init(Database database) { - return SqlUtilsBuilder - .getBuilder() + return SqlUtilsBuilder.getBuilder() .withName(database.getId() + AT_SYMBOL + database.getName()) .withType(database.getType()) .withJdbcUrl(database.getUrl()) @@ -113,25 +105,27 @@ public class SqlUtils { getResult(sql, queryResultWithColumns, jdbcTemplate()); } - private SemanticQueryResp getResult(String sql, SemanticQueryResp queryResultWithColumns, - JdbcTemplate jdbcTemplate) { - jdbcTemplate.query(sql, rs -> { - if (null == rs) { - return queryResultWithColumns; - } + private SemanticQueryResp getResult( + String sql, SemanticQueryResp queryResultWithColumns, JdbcTemplate jdbcTemplate) { + jdbcTemplate.query( + sql, + rs -> { + if (null == rs) { + return queryResultWithColumns; + } - ResultSetMetaData metaData = rs.getMetaData(); - List queryColumns = new ArrayList<>(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String key = metaData.getColumnLabel(i); - queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i))); - } - queryResultWithColumns.setColumns(queryColumns); + ResultSetMetaData metaData = rs.getMetaData(); + List queryColumns = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String key = metaData.getColumnLabel(i); + queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i))); + } + queryResultWithColumns.setColumns(queryColumns); - List> resultList = getAllData(rs, queryColumns); - queryResultWithColumns.setResultList(resultList); - return queryResultWithColumns; - }); + List> resultList = getAllData(rs, queryColumns); + queryResultWithColumns.setResultList(resultList); + return queryResultWithColumns; + }); return queryResultWithColumns; } @@ -147,7 +141,8 @@ public class SqlUtils { return data; } - private Map getLineData(ResultSet rs, List queryColumns) throws SQLException { + private Map getLineData(ResultSet rs, List queryColumns) + throws SQLException { Map map = new LinkedHashMap<>(); for (QueryColumn queryColumn : queryColumns) { String colName = queryColumn.getNameEn(); @@ -184,9 +179,7 @@ public class SqlUtils { private String username; private String password; - private SqlUtilsBuilder() { - - } + private SqlUtilsBuilder() {} public static SqlUtilsBuilder getBuilder() { return new SqlUtilsBuilder(); @@ -233,13 +226,14 @@ public class SqlUtils { } public SqlUtils build() { - Database database = Database.builder() - .name(this.name) - .type(this.type) - .url(this.jdbcUrl) - .username(this.username) - .password(this.password) - .build(); + Database database = + Database.builder() + .name(this.name) + .type(this.type) + .url(this.jdbcUrl) + .username(this.username) + .password(this.password) + .build(); SqlUtils sqlUtils = new SqlUtils(database); sqlUtils.jdbcDataSource = this.jdbcDataSource; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java index 2a38bc383..6a3d0ae78 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; import org.stringtemplate.v4.ST; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -15,14 +16,16 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; + import static com.tencent.supersonic.common.pojo.Constants.COMMA; import static com.tencent.supersonic.common.pojo.Constants.EMPTY; @Slf4j public class SqlVariableParseUtils { - public static final String REG_SENSITIVE_SQL = "drop\\s|alter\\s|grant\\s|insert\\s|replace\\s|delete\\s|" - + "truncate\\s|update\\s|remove\\s"; + public static final String REG_SENSITIVE_SQL = + "drop\\s|alter\\s|grant\\s|insert\\s|replace\\s|delete\\s|" + + "truncate\\s|update\\s|remove\\s"; public static final Pattern PATTERN_SENSITIVE_SQL = Pattern.compile(REG_SENSITIVE_SQL); public static final String APOSTROPHE = "'"; @@ -34,33 +37,39 @@ public class SqlVariableParseUtils { if (CollectionUtils.isEmpty(sqlVariables)) { return sql; } - //1. handle default variable value - sqlVariables.forEach(variable -> { - variables.put(variable.getName().trim(), - getValues(variable.getValueType(), variable.getDefaultValues())); - }); + // 1. handle default variable value + sqlVariables.forEach( + variable -> { + variables.put( + variable.getName().trim(), + getValues(variable.getValueType(), variable.getDefaultValues())); + }); - //override by variable param + // override by variable param if (!CollectionUtils.isEmpty(params)) { Map> map = sqlVariables.stream().collect(Collectors.groupingBy(SqlVariable::getName)); - params.forEach(p -> { - if (map.containsKey(p.getName())) { - List list = map.get(p.getName()); - if (!CollectionUtils.isEmpty(list)) { - SqlVariable v = list.get(list.size() - 1); - variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue())); - } - } - }); + params.forEach( + p -> { + if (map.containsKey(p.getName())) { + List list = map.get(p.getName()); + if (!CollectionUtils.isEmpty(list)) { + SqlVariable v = list.get(list.size() - 1); + variables.put( + p.getName().trim(), + getValue(v.getValueType(), p.getValue())); + } + } + }); } - variables.forEach((k, v) -> { - if (v instanceof List && ((List) v).size() > 0) { - v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString(); - } - variables.put(k, v); - }); + variables.forEach( + (k, v) -> { + if (v instanceof List && ((List) v).size() > 0) { + v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString(); + } + variables.put(k, v); + }); return parse(sql, variables); } @@ -79,12 +88,18 @@ public class SqlVariableParseUtils { if (null != valueType) { switch (valueType) { case STRING: - return values.stream().map(String::valueOf) - .map(s -> s.startsWith(APOSTROPHE) && s.endsWith(APOSTROPHE) - ? s : String.join(EMPTY, APOSTROPHE, s, APOSTROPHE)) + return values.stream() + .map(String::valueOf) + .map( + s -> + s.startsWith(APOSTROPHE) && s.endsWith(APOSTROPHE) + ? s + : String.join(EMPTY, APOSTROPHE, s, APOSTROPHE)) .collect(Collectors.toList()); case EXPR: - values.stream().map(String::valueOf).forEach(SqlVariableParseUtils::checkSensitiveSql); + values.stream() + .map(String::valueOf) + .forEach(SqlVariableParseUtils::checkSensitiveSql); return values.stream().map(String::valueOf).collect(Collectors.toList()); case NUMBER: return values.stream().map(String::valueOf).collect(Collectors.toList()); @@ -100,8 +115,11 @@ public class SqlVariableParseUtils { if (null != valueType) { switch (valueType) { case STRING: - return String.join(EMPTY, value.startsWith(APOSTROPHE) ? EMPTY : APOSTROPHE, - value, value.endsWith(APOSTROPHE) ? EMPTY : APOSTROPHE); + return String.join( + EMPTY, + value.startsWith(APOSTROPHE) ? EMPTY : APOSTROPHE, + value, + value.endsWith(APOSTROPHE) ? EMPTY : APOSTROPHE); case NUMBER: case EXPR: default: @@ -117,8 +135,8 @@ public class SqlVariableParseUtils { if (matcher.find()) { String group = matcher.group(); log.warn("Sensitive SQL operations are not allowed: {}", group.toUpperCase()); - throw new InvalidArgumentException("Sensitive SQL operations are not allowed: " + group.toUpperCase()); + throw new InvalidArgumentException( + "Sensitive SQL operations are not allowed: " + group.toUpperCase()); } } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java index 0ac9b99a3..7dfece732 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import lombok.extern.slf4j.Slf4j; @@ -11,15 +11,14 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** - * generate system time dimension tools - */ +/** generate system time dimension tools */ @Slf4j public class SysTimeDimensionBuilder { // Defines the regular expression pattern for the time keyword private static final Pattern TIME_KEYWORD_PATTERN = - Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b", + Pattern.compile( + "\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b", Pattern.CASE_INSENSITIVE); public static void addSysTimeDimension(List dims, DbAdaptor engineAdaptor) { @@ -27,7 +26,7 @@ public class SysTimeDimensionBuilder { Dim timeDim = getTimeDim(dims); if (timeDim == null) { timeDim = Dim.getDefault(); - //todo not find the time dimension + // todo not find the time dimension return; } dims.add(generateSysDayDimension(timeDim, engineAdaptor)); @@ -40,7 +39,9 @@ public class SysTimeDimensionBuilder { Dim dim = new Dim(); dim.setBizName(TimeDimensionEnum.DAY.getName()); dim.setType(DimensionType.partition_time.name()); - dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(), engineAdaptor)); + dim.setExpr( + generateTimeExpr( + timeDim, TimeDimensionEnum.DAY.name().toLowerCase(), engineAdaptor)); DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams(); typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase()); typeParams.setIsPrimary("true"); @@ -52,7 +53,9 @@ public class SysTimeDimensionBuilder { Dim dim = new Dim(); dim.setBizName(TimeDimensionEnum.WEEK.getName()); dim.setType(DimensionType.partition_time.name()); - dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(), engineAdaptor)); + dim.setExpr( + generateTimeExpr( + timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(), engineAdaptor)); DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams(); typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase()); typeParams.setIsPrimary("false"); @@ -64,7 +67,9 @@ public class SysTimeDimensionBuilder { Dim dim = new Dim(); dim.setBizName(TimeDimensionEnum.MONTH.getName()); dim.setType(DimensionType.partition_time.name()); - dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(), engineAdaptor)); + dim.setExpr( + generateTimeExpr( + timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(), engineAdaptor)); DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams(); typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase()); typeParams.setIsPrimary("false"); @@ -87,7 +92,6 @@ public class SysTimeDimensionBuilder { } else { return engineAdaptor.getDateFormat(dateType, dateFormat, bizName); } - } private static Dim getTimeDim(List timeDims) { @@ -98,5 +102,4 @@ public class SysTimeDimensionBuilder { } return null; } - } diff --git a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java index 0e9d372a6..30ee19cf7 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.core.parser.aggregate; import com.alibaba.fastjson.JSON; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser; import org.junit.jupiter.api.Test; import org.testng.Assert; @@ -11,460 +11,462 @@ public class CalciteSqlParserTest { @Test public void testCalciteSqlParser() throws Exception { - String json = "{\n" - + " \"dataSetId\": 1,\n" - + " \"sql\": \"\",\n" - + " \"sourceId\": \"\",\n" - + " \"errMsg\": \"\",\n" - + " \"metricQueryParam\": {\n" - + " \"metrics\": [\n" - + " \"pv\"\n" - + " ],\n" - + " \"dimensions\": [\n" - + " \"sys_imp_date\"\n" - + " ],\n" - + " \"nativeQuery\": false\n" - + " },\n" - + " \"status\": 0,\n" - + " \"isS2SQL\": false,\n" - + " \"enableOptimize\": true,\n" - + " \"minMaxTime\": {\n" - + " \"left\": \"sys_imp_date\",\n" - + " \"middle\": \"2024-03-24\",\n" - + " \"right\": \"2024-03-18\"\n" - + " },\n" - + " \"dataSetSql\": \"SELECT sys_imp_date, SUM(pv) AS pv FROM t_1 WHERE " - + "sys_imp_date >= '2024-03-18' AND sys_imp_date <= '2024-03-24' GROUP BY sys_imp_date LIMIT 365\",\n" - + " \"dataSetAlias\": \"t_1\",\n" - + " \"dataSetSimplifySql\": \"\",\n" - + " \"enableLimitWrapper\": false,\n" - + " \"semanticModel\": {\n" - + " \"schemaKey\": \"VIEW_1\",\n" - + " \"metrics\": [\n" - + " {\n" - + " \"name\": \"pv\",\n" - + " \"owners\": [\n" - + " \"admin\"\n" - + " ],\n" - + " \"type\": \"ATOMIC\",\n" - + " \"metricTypeParams\": {\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"s2_pv_uv_statis_pv\",\n" - + " \"agg\": \"SUM\",\n" - + " \"constraint\": \"\"\n" - + " }\n" - + " ],\n" - + " \"isFieldMetric\": false,\n" - + " \"expr\": \"s2_pv_uv_statis_pv\"\n" - + " }\n" - + " },\n" - + " {\n" - + " \"name\": \"uv\",\n" - + " \"owners\": [\n" - + " \"admin\"\n" - + " ],\n" - + " \"type\": \"DERIVED\",\n" - + " \"metricTypeParams\": {\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"user_id\",\n" - + " \"expr\": \"user_id\"\n" - + " }\n" - + " ],\n" - + " \"isFieldMetric\": true,\n" - + " \"expr\": \"user_id\"\n" - + " }\n" - + " },\n" - + " {\n" - + " \"name\": \"pv_avg\",\n" - + " \"owners\": [\n" - + " \"admin\"\n" - + " ],\n" - + " \"type\": \"DERIVED\",\n" - + " \"metricTypeParams\": {\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"pv\",\n" - + " \"expr\": \"pv\"\n" - + " },\n" - + " {\n" - + " \"name\": \"uv\",\n" - + " \"expr\": \"uv\"\n" - + " }\n" - + " ],\n" - + " \"isFieldMetric\": true,\n" - + " \"expr\": \"pv\"\n" - + " }\n" - + " },\n" - + " {\n" - + " \"name\": \"stay_hours\",\n" - + " \"owners\": [\n" - + " \"admin\"\n" - + " ],\n" - + " \"type\": \"ATOMIC\",\n" - + " \"metricTypeParams\": {\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"s2_stay_time_statis_stay_hours\",\n" - + " \"agg\": \"SUM\",\n" - + " \"constraint\": \"\"\n" - + " }\n" - + " ],\n" - + " \"isFieldMetric\": false,\n" - + " \"expr\": \"s2_stay_time_statis_stay_hours\"\n" - + " }\n" - + " }\n" - + " ],\n" - + " \"datasourceMap\": {\n" - + " \"user_department\": {\n" - + " \"id\": 1,\n" - + " \"name\": \"user_department\",\n" - + " \"sourceId\": 1,\n" - + " \"type\": \"h2\",\n" - + " \"sqlQuery\": \"select user_name,department from s2_user_department\",\n" - + " \"identifiers\": [\n" - + " {\n" - + " \"name\": \"user_name\",\n" - + " \"type\": \"primary\"\n" - + " }\n" - + " ],\n" - + " \"dimensions\": [\n" - + " {\n" - + " \"name\": \"department\",\n" - + " \"type\": \"categorical\",\n" - + " \"expr\": \"department\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"department\"\n" - + " }\n" - + " ],\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"user_department_internal_cnt\",\n" - + " \"agg\": \"count\",\n" - + " \"expr\": \"user_name\"\n" - + " },\n" - + " {\n" - + " \"name\": \"user_name\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"user_name\"\n" - + " },\n" - + " {\n" - + " \"name\": \"department\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"department\"\n" - + " }\n" - + " ],\n" - + " \"aggTime\": \"none\"\n" - + " },\n" - + " \"s2_pv_uv_statis\": {\n" - + " \"id\": 2,\n" - + " \"name\": \"s2_pv_uv_statis\",\n" - + " \"sourceId\": 1,\n" - + " \"type\": \"h2\",\n" - + " \"sqlQuery\": \"SELECT imp_date, user_name, page, 1 as pv, user_name as user_id " - + "FROM s2_pv_uv_statis\",\n" - + " \"identifiers\": [\n" - + " {\n" - + " \"name\": \"user_name\",\n" - + " \"type\": \"primary\"\n" - + " }\n" - + " ],\n" - + " \"dimensions\": [\n" - + " {\n" - + " \"name\": \"imp_date\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"imp_date\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"true\",\n" - + " \"timeGranularity\": \"day\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"imp_date\"\n" - + " },\n" - + " {\n" - + " \"name\": \"page\",\n" - + " \"type\": \"categorical\",\n" - + " \"expr\": \"page\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"page\"\n" - + " },\n" - + " {\n" - + " \"name\": \"sys_imp_date\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"imp_date\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"true\",\n" - + " \"timeGranularity\": \"day\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"sys_imp_date\"\n" - + " },\n" - + " {\n" - + " \"name\": \"sys_imp_week\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"false\",\n" - + " \"timeGranularity\": \"week\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"sys_imp_week\"\n" - + " },\n" - + " {\n" - + " \"name\": \"sys_imp_month\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"FORMATDATETIME(PARSEDATETIME" - + "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"false\",\n" - + " \"timeGranularity\": \"month\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"sys_imp_month\"\n" - + " }\n" - + " ],\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"s2_pv_uv_statis_pv\",\n" - + " \"agg\": \"SUM\",\n" - + " \"expr\": \"pv\"\n" - + " },\n" - + " {\n" - + " \"name\": \"s2_pv_uv_statis_user_id\",\n" - + " \"agg\": \"SUM\",\n" - + " \"expr\": \"user_id\"\n" - + " },\n" - + " {\n" - + " \"name\": \"s2_pv_uv_statis_internal_cnt\",\n" - + " \"agg\": \"count\",\n" - + " \"expr\": \"user_name\"\n" - + " },\n" - + " {\n" - + " \"name\": \"user_name\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"user_name\"\n" - + " },\n" - + " {\n" - + " \"name\": \"imp_date\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"imp_date\"\n" - + " },\n" - + " {\n" - + " \"name\": \"page\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"page\"\n" - + " },\n" - + " {\n" - + " \"name\": \"pv\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"pv\"\n" - + " },\n" - + " {\n" - + " \"name\": \"user_id\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"user_id\"\n" - + " }\n" - + " ],\n" - + " \"aggTime\": \"day\"\n" - + " },\n" - + " \"s2_stay_time_statis\": {\n" - + " \"id\": 3,\n" - + " \"name\": \"s2_stay_time_statis\",\n" - + " \"sourceId\": 1,\n" - + " \"type\": \"h2\",\n" - + " \"sqlQuery\": \"select imp_date,user_name,stay_hours" - + ",page from s2_stay_time_statis\",\n" - + " \"identifiers\": [\n" - + " {\n" - + " \"name\": \"user_name\",\n" - + " \"type\": \"primary\"\n" - + " }\n" - + " ],\n" - + " \"dimensions\": [\n" - + " {\n" - + " \"name\": \"imp_date\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"imp_date\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"true\",\n" - + " \"timeGranularity\": \"day\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"imp_date\"\n" - + " },\n" - + " {\n" - + " \"name\": \"page\",\n" - + " \"type\": \"categorical\",\n" - + " \"expr\": \"page\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"page\"\n" - + " },\n" - + " {\n" - + " \"name\": \"sys_imp_date\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"imp_date\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"true\",\n" - + " \"timeGranularity\": \"day\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"sys_imp_date\"\n" - + " },\n" - + " {\n" - + " \"name\": \"sys_imp_week\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"false\",\n" - + " \"timeGranularity\": \"week\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"sys_imp_week\"\n" - + " },\n" - + " {\n" - + " \"name\": \"sys_imp_month\",\n" - + " \"type\": \"time\",\n" - + " \"expr\": \"FORMATDATETIME(PARSEDATETIME" - + "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " \"isPrimary\": \"false\",\n" - + " \"timeGranularity\": \"month\"\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"sys_imp_month\"\n" - + " }\n" - + " ],\n" - + " \"measures\": [\n" - + " {\n" - + " \"name\": \"s2_stay_time_statis_stay_hours\",\n" - + " \"agg\": \"SUM\",\n" - + " \"expr\": \"stay_hours\"\n" - + " },\n" - + " {\n" - + " \"name\": \"s2_stay_time_statis_internal_cnt\",\n" - + " \"agg\": \"count\",\n" - + " \"expr\": \"user_name\"\n" - + " },\n" - + " {\n" - + " \"name\": \"user_name\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"user_name\"\n" - + " },\n" - + " {\n" - + " \"name\": \"imp_date\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"imp_date\"\n" - + " },\n" - + " {\n" - + " \"name\": \"page\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"page\"\n" - + " },\n" - + " {\n" - + " \"name\": \"stay_hours\",\n" - + " \"agg\": \"\",\n" - + " \"expr\": \"stay_hours\"\n" - + " }\n" - + " ],\n" - + " \"aggTime\": \"day\"\n" - + " }\n" - + " },\n" - + " \"dimensionMap\": {\n" - + " \"user_department\": [\n" - + " {\n" - + " \"name\": \"department\",\n" - + " \"owners\": \"admin\",\n" - + " \"type\": \"categorical\",\n" - + " \"expr\": \"department\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"department\"\n" - + " }\n" - + " ],\n" - + " \"s2_pv_uv_statis\": [\n" - + " ],\n" - + " \"s2_stay_time_statis\": [\n" - + " {\n" - + " \"name\": \"page\",\n" - + " \"owners\": \"admin\",\n" - + " \"type\": \"categorical\",\n" - + " \"expr\": \"page\",\n" - + " \"dimensionTimeTypeParams\": {\n" - + " },\n" - + " \"dataType\": \"UNKNOWN\",\n" - + " \"bizName\": \"page\"\n" - + " }\n" - + " ]\n" - + " },\n" - + " \"materializationList\": [\n" - + " ],\n" - + " \"joinRelations\": [\n" - + " {\n" - + " \"id\": 1,\n" - + " \"left\": \"user_department\",\n" - + " \"right\": \"s2_pv_uv_statis\",\n" - + " \"joinType\": \"left join\",\n" - + " \"joinCondition\": [\n" - + " {\n" - + " \"left\": \"user_name\",\n" - + " \"middle\": \"=\",\n" - + " \"right\": \"user_name\"\n" - + " }\n" - + " ]\n" - + " },\n" - + " {\n" - + " \"id\": 2,\n" - + " \"left\": \"user_department\",\n" - + " \"right\": \"s2_stay_time_statis\",\n" - + " \"joinType\": \"left join\",\n" - + " \"joinCondition\": [\n" - + " {\n" - + " \"left\": \"user_name\",\n" - + " \"middle\": \"=\",\n" - + " \"right\": \"user_name\"\n" - + " }\n" - + " ]\n" - + " }\n" - + " ],\n" - + " \"database\": {\n" - + " \"id\": 1,\n" - + " \"name\": \"数据实例\",\n" - + " \"description\": \"样例数据库实例\",\n" - + " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n" - + " \"username\": \"root\",\n" - + " \"password\": \"semantic\",\n" - + " \"type\": \"h2\",\n" - + " \"connectInfo\": {\n" - + " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n" - + " \"userName\": \"root\",\n" - + " \"password\": \"semantic\"\n" - + " },\n" - + " \"admins\": [\n" - + " ],\n" - + " \"viewers\": [\n" - + " ],\n" - + " \"createdBy\": \"admin\",\n" - + " \"updatedBy\": \"admin\",\n" - + " \"createdAt\": 1711367511146,\n" - + " \"updatedAt\": 1711367511146\n" - + " }\n" - + " }\n" - + "}"; + String json = + "{\n" + + " \"dataSetId\": 1,\n" + + " \"sql\": \"\",\n" + + " \"sourceId\": \"\",\n" + + " \"errMsg\": \"\",\n" + + " \"metricQueryParam\": {\n" + + " \"metrics\": [\n" + + " \"pv\"\n" + + " ],\n" + + " \"dimensions\": [\n" + + " \"sys_imp_date\"\n" + + " ],\n" + + " \"nativeQuery\": false\n" + + " },\n" + + " \"status\": 0,\n" + + " \"isS2SQL\": false,\n" + + " \"enableOptimize\": true,\n" + + " \"minMaxTime\": {\n" + + " \"left\": \"sys_imp_date\",\n" + + " \"middle\": \"2024-03-24\",\n" + + " \"right\": \"2024-03-18\"\n" + + " },\n" + + " \"dataSetSql\": \"SELECT sys_imp_date, SUM(pv) AS pv FROM t_1 WHERE " + + "sys_imp_date >= '2024-03-18' AND sys_imp_date <= '2024-03-24' GROUP BY sys_imp_date LIMIT 365\",\n" + + " \"dataSetAlias\": \"t_1\",\n" + + " \"dataSetSimplifySql\": \"\",\n" + + " \"enableLimitWrapper\": false,\n" + + " \"semanticModel\": {\n" + + " \"schemaKey\": \"VIEW_1\",\n" + + " \"metrics\": [\n" + + " {\n" + + " \"name\": \"pv\",\n" + + " \"owners\": [\n" + + " \"admin\"\n" + + " ],\n" + + " \"type\": \"ATOMIC\",\n" + + " \"metricTypeParams\": {\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"s2_pv_uv_statis_pv\",\n" + + " \"agg\": \"SUM\",\n" + + " \"constraint\": \"\"\n" + + " }\n" + + " ],\n" + + " \"isFieldMetric\": false,\n" + + " \"expr\": \"s2_pv_uv_statis_pv\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"uv\",\n" + + " \"owners\": [\n" + + " \"admin\"\n" + + " ],\n" + + " \"type\": \"DERIVED\",\n" + + " \"metricTypeParams\": {\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"user_id\",\n" + + " \"expr\": \"user_id\"\n" + + " }\n" + + " ],\n" + + " \"isFieldMetric\": true,\n" + + " \"expr\": \"user_id\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"pv_avg\",\n" + + " \"owners\": [\n" + + " \"admin\"\n" + + " ],\n" + + " \"type\": \"DERIVED\",\n" + + " \"metricTypeParams\": {\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"pv\",\n" + + " \"expr\": \"pv\"\n" + + " },\n" + + " {\n" + + " \"name\": \"uv\",\n" + + " \"expr\": \"uv\"\n" + + " }\n" + + " ],\n" + + " \"isFieldMetric\": true,\n" + + " \"expr\": \"pv\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"stay_hours\",\n" + + " \"owners\": [\n" + + " \"admin\"\n" + + " ],\n" + + " \"type\": \"ATOMIC\",\n" + + " \"metricTypeParams\": {\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"s2_stay_time_statis_stay_hours\",\n" + + " \"agg\": \"SUM\",\n" + + " \"constraint\": \"\"\n" + + " }\n" + + " ],\n" + + " \"isFieldMetric\": false,\n" + + " \"expr\": \"s2_stay_time_statis_stay_hours\"\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"datasourceMap\": {\n" + + " \"user_department\": {\n" + + " \"id\": 1,\n" + + " \"name\": \"user_department\",\n" + + " \"sourceId\": 1,\n" + + " \"type\": \"h2\",\n" + + " \"sqlQuery\": \"select user_name,department from s2_user_department\",\n" + + " \"identifiers\": [\n" + + " {\n" + + " \"name\": \"user_name\",\n" + + " \"type\": \"primary\"\n" + + " }\n" + + " ],\n" + + " \"dimensions\": [\n" + + " {\n" + + " \"name\": \"department\",\n" + + " \"type\": \"categorical\",\n" + + " \"expr\": \"department\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"department\"\n" + + " }\n" + + " ],\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"user_department_internal_cnt\",\n" + + " \"agg\": \"count\",\n" + + " \"expr\": \"user_name\"\n" + + " },\n" + + " {\n" + + " \"name\": \"user_name\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"user_name\"\n" + + " },\n" + + " {\n" + + " \"name\": \"department\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"department\"\n" + + " }\n" + + " ],\n" + + " \"aggTime\": \"none\"\n" + + " },\n" + + " \"s2_pv_uv_statis\": {\n" + + " \"id\": 2,\n" + + " \"name\": \"s2_pv_uv_statis\",\n" + + " \"sourceId\": 1,\n" + + " \"type\": \"h2\",\n" + + " \"sqlQuery\": \"SELECT imp_date, user_name, page, 1 as pv, user_name as user_id " + + "FROM s2_pv_uv_statis\",\n" + + " \"identifiers\": [\n" + + " {\n" + + " \"name\": \"user_name\",\n" + + " \"type\": \"primary\"\n" + + " }\n" + + " ],\n" + + " \"dimensions\": [\n" + + " {\n" + + " \"name\": \"imp_date\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"imp_date\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"true\",\n" + + " \"timeGranularity\": \"day\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"imp_date\"\n" + + " },\n" + + " {\n" + + " \"name\": \"page\",\n" + + " \"type\": \"categorical\",\n" + + " \"expr\": \"page\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"page\"\n" + + " },\n" + + " {\n" + + " \"name\": \"sys_imp_date\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"imp_date\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"true\",\n" + + " \"timeGranularity\": \"day\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"sys_imp_date\"\n" + + " },\n" + + " {\n" + + " \"name\": \"sys_imp_week\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"false\",\n" + + " \"timeGranularity\": \"week\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"sys_imp_week\"\n" + + " },\n" + + " {\n" + + " \"name\": \"sys_imp_month\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"FORMATDATETIME(PARSEDATETIME" + + "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"false\",\n" + + " \"timeGranularity\": \"month\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"sys_imp_month\"\n" + + " }\n" + + " ],\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"s2_pv_uv_statis_pv\",\n" + + " \"agg\": \"SUM\",\n" + + " \"expr\": \"pv\"\n" + + " },\n" + + " {\n" + + " \"name\": \"s2_pv_uv_statis_user_id\",\n" + + " \"agg\": \"SUM\",\n" + + " \"expr\": \"user_id\"\n" + + " },\n" + + " {\n" + + " \"name\": \"s2_pv_uv_statis_internal_cnt\",\n" + + " \"agg\": \"count\",\n" + + " \"expr\": \"user_name\"\n" + + " },\n" + + " {\n" + + " \"name\": \"user_name\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"user_name\"\n" + + " },\n" + + " {\n" + + " \"name\": \"imp_date\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"imp_date\"\n" + + " },\n" + + " {\n" + + " \"name\": \"page\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"page\"\n" + + " },\n" + + " {\n" + + " \"name\": \"pv\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"pv\"\n" + + " },\n" + + " {\n" + + " \"name\": \"user_id\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"user_id\"\n" + + " }\n" + + " ],\n" + + " \"aggTime\": \"day\"\n" + + " },\n" + + " \"s2_stay_time_statis\": {\n" + + " \"id\": 3,\n" + + " \"name\": \"s2_stay_time_statis\",\n" + + " \"sourceId\": 1,\n" + + " \"type\": \"h2\",\n" + + " \"sqlQuery\": \"select imp_date,user_name,stay_hours" + + ",page from s2_stay_time_statis\",\n" + + " \"identifiers\": [\n" + + " {\n" + + " \"name\": \"user_name\",\n" + + " \"type\": \"primary\"\n" + + " }\n" + + " ],\n" + + " \"dimensions\": [\n" + + " {\n" + + " \"name\": \"imp_date\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"imp_date\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"true\",\n" + + " \"timeGranularity\": \"day\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"imp_date\"\n" + + " },\n" + + " {\n" + + " \"name\": \"page\",\n" + + " \"type\": \"categorical\",\n" + + " \"expr\": \"page\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"page\"\n" + + " },\n" + + " {\n" + + " \"name\": \"sys_imp_date\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"imp_date\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"true\",\n" + + " \"timeGranularity\": \"day\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"sys_imp_date\"\n" + + " },\n" + + " {\n" + + " \"name\": \"sys_imp_week\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"false\",\n" + + " \"timeGranularity\": \"week\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"sys_imp_week\"\n" + + " },\n" + + " {\n" + + " \"name\": \"sys_imp_month\",\n" + + " \"type\": \"time\",\n" + + " \"expr\": \"FORMATDATETIME(PARSEDATETIME" + + "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " \"isPrimary\": \"false\",\n" + + " \"timeGranularity\": \"month\"\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"sys_imp_month\"\n" + + " }\n" + + " ],\n" + + " \"measures\": [\n" + + " {\n" + + " \"name\": \"s2_stay_time_statis_stay_hours\",\n" + + " \"agg\": \"SUM\",\n" + + " \"expr\": \"stay_hours\"\n" + + " },\n" + + " {\n" + + " \"name\": \"s2_stay_time_statis_internal_cnt\",\n" + + " \"agg\": \"count\",\n" + + " \"expr\": \"user_name\"\n" + + " },\n" + + " {\n" + + " \"name\": \"user_name\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"user_name\"\n" + + " },\n" + + " {\n" + + " \"name\": \"imp_date\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"imp_date\"\n" + + " },\n" + + " {\n" + + " \"name\": \"page\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"page\"\n" + + " },\n" + + " {\n" + + " \"name\": \"stay_hours\",\n" + + " \"agg\": \"\",\n" + + " \"expr\": \"stay_hours\"\n" + + " }\n" + + " ],\n" + + " \"aggTime\": \"day\"\n" + + " }\n" + + " },\n" + + " \"dimensionMap\": {\n" + + " \"user_department\": [\n" + + " {\n" + + " \"name\": \"department\",\n" + + " \"owners\": \"admin\",\n" + + " \"type\": \"categorical\",\n" + + " \"expr\": \"department\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"department\"\n" + + " }\n" + + " ],\n" + + " \"s2_pv_uv_statis\": [\n" + + " ],\n" + + " \"s2_stay_time_statis\": [\n" + + " {\n" + + " \"name\": \"page\",\n" + + " \"owners\": \"admin\",\n" + + " \"type\": \"categorical\",\n" + + " \"expr\": \"page\",\n" + + " \"dimensionTimeTypeParams\": {\n" + + " },\n" + + " \"dataType\": \"UNKNOWN\",\n" + + " \"bizName\": \"page\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"materializationList\": [\n" + + " ],\n" + + " \"joinRelations\": [\n" + + " {\n" + + " \"id\": 1,\n" + + " \"left\": \"user_department\",\n" + + " \"right\": \"s2_pv_uv_statis\",\n" + + " \"joinType\": \"left join\",\n" + + " \"joinCondition\": [\n" + + " {\n" + + " \"left\": \"user_name\",\n" + + " \"middle\": \"=\",\n" + + " \"right\": \"user_name\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"id\": 2,\n" + + " \"left\": \"user_department\",\n" + + " \"right\": \"s2_stay_time_statis\",\n" + + " \"joinType\": \"left join\",\n" + + " \"joinCondition\": [\n" + + " {\n" + + " \"left\": \"user_name\",\n" + + " \"middle\": \"=\",\n" + + " \"right\": \"user_name\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"database\": {\n" + + " \"id\": 1,\n" + + " \"name\": \"数据实例\",\n" + + " \"description\": \"样例数据库实例\",\n" + + " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n" + + " \"username\": \"root\",\n" + + " \"password\": \"semantic\",\n" + + " \"type\": \"h2\",\n" + + " \"connectInfo\": {\n" + + " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n" + + " \"userName\": \"root\",\n" + + " \"password\": \"semantic\"\n" + + " },\n" + + " \"admins\": [\n" + + " ],\n" + + " \"viewers\": [\n" + + " ],\n" + + " \"createdBy\": \"admin\",\n" + + " \"updatedBy\": \"admin\",\n" + + " \"createdAt\": 1711367511146,\n" + + " \"updatedAt\": 1711367511146\n" + + " }\n" + + " }\n" + + "}"; QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class); CalciteQueryParser calciteSqlParser = new CalciteQueryParser(); calciteSqlParser.parse(queryStatement, AggOption.DEFAULT); - Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""), + Assert.assertEquals( + queryStatement.getSql().trim().replaceAll("\\s+", ""), "SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" - + "FROM" - + "`s2_pv_uv_statis`" - + "GROUPBY`imp_date`,`imp_date`"); + + "FROM" + + "`s2_pv_uv_statis`" + + "GROUPBY`imp_date`,`imp_date`"); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java index 3b53782ff..195b6d813 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java @@ -7,6 +7,4 @@ import java.lang.annotation.Target; @Target({ElementType.PARAMETER, ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) -public @interface ApiHeaderCheck { - -} +public @interface ApiHeaderCheck {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2DataPermission.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2DataPermission.java index 3eaba07f8..4979d2369 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2DataPermission.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2DataPermission.java @@ -1,14 +1,12 @@ package com.tencent.supersonic.headless.server.annotation; -import java.lang.annotation.Target; -import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Documented; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; -import java.lang.annotation.Documented; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) @Documented -public @interface S2DataPermission { - -} +public @interface S2DataPermission {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java index 65656eb37..6fc126fdd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.aspect; +import javax.servlet.http.HttpServletRequest; + import com.tencent.supersonic.common.pojo.Pair; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.SignatureUtils; @@ -16,8 +18,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import javax.servlet.http.HttpServletRequest; - @Component @Aspect @Order(1) @@ -30,12 +30,10 @@ public class ApiHeaderCheckAspect { private static final String SIGNATURE = "signature"; - @Autowired - private AppService appService; + @Autowired private AppService appService; @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck)") - private void apiPermissionCheck() { - } + private void apiPermissionCheck() {} @Around("apiPermissionCheck()") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { @@ -65,8 +63,12 @@ public class ApiHeaderCheckAspect { if (!AppStatus.ONLINE.equals(appDetailResp.getAppStatus())) { throw new InvalidArgumentException("该应用暂时为非在线状态"); } - Pair checkResult = SignatureUtils.isValidSignature(appId, appDetailResp.getAppSecret(), - Long.parseLong(timestampStr), signature); + Pair checkResult = + SignatureUtils.isValidSignature( + appId, + appDetailResp.getAppSecret(), + Long.parseLong(timestampStr), + signature); if (!checkResult.first) { throw new InvalidArgumentException(checkResult.second); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 74bb936eb..1edeb460c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -10,13 +10,13 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -43,10 +43,11 @@ public class DimValueAspect { @Value("${s2.dimension.value.map.enable:true}") private Boolean dimensionValueMapEnable; - @Autowired - private DimensionService dimensionService; - @Around("execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))") + @Autowired private DimensionService dimensionService; + + @Around( + "execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))") public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (!dimensionValueMapEnable) { log.debug("dimensionValueMapEnable is false, skip dimensionValueMap"); @@ -70,8 +71,10 @@ public class DimValueAspect { QueryStructReq queryStructReq = (QueryStructReq) args[0]; MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds())); List dimensions = dimensionService.getDimensions(metaFilter); - Map> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions); - Map> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions); + Map> dimAndAliasAndTechNamePair = + getAliasAndBizNameToTechName(dimensions); + Map> dimAndTechNameAndBizNamePair = + getTechNameToBizName(dimensions); rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair); @@ -91,7 +94,8 @@ public class DimValueAspect { log.debug("correctorSql before replacing:{}", sql); List fieldExpressionList = SqlSelectHelper.getWhereExpressions(sql); List dimensions = dimensionService.getDimensions(metaFilter); - Set fieldNames = dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet()); + Set fieldNames = + dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet()); Map> filedNameToValueMap = new HashMap<>(); for (FieldExpression expression : fieldExpressionList) { if (!fieldNames.contains(expression.getFieldName())) { @@ -104,13 +108,22 @@ public class DimValueAspect { } // consider '=' filter if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) { - dimension.getDimValueMaps().stream().forEach(dimValue -> { - if (!CollectionUtils.isEmpty(dimValue.getAlias()) - && dimValue.getAlias().contains(expression.getFieldValue().toString())) { - getFiledNameToValueMap(filedNameToValueMap, expression.getFieldValue().toString(), - dimValue.getTechName(), expression.getFieldName()); - } - }); + dimension.getDimValueMaps().stream() + .forEach( + dimValue -> { + if (!CollectionUtils.isEmpty(dimValue.getAlias()) + && dimValue.getAlias() + .contains( + expression + .getFieldValue() + .toString())) { + getFiledNameToValueMap( + filedNameToValueMap, + expression.getFieldValue().toString(), + dimValue.getTechName(), + expression.getFieldName()); + } + }); } // consider 'in' filter,each element needs to judge. replaceInCondition(expression, dimension, filedNameToValueMap); @@ -128,7 +141,9 @@ public class DimValueAspect { return queryResultWithColumns; } - public void replaceInCondition(FieldExpression expression, DimensionResp dimension, + public void replaceInCondition( + FieldExpression expression, + DimensionResp dimension, Map> filedNameToValueMap) { if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) { String fieldValue = JsonUtil.toString(expression.getFieldValue()); @@ -150,20 +165,27 @@ public class DimValueAspect { } } if (!revisedValues.equals(values)) { - getFiledNameToValueMap(filedNameToValueMap, JsonUtil.toString(values), - JsonUtil.toString(revisedValues), expression.getFieldName()); + getFiledNameToValueMap( + filedNameToValueMap, + JsonUtil.toString(values), + JsonUtil.toString(revisedValues), + expression.getFieldName()); } } } - public void getFiledNameToValueMap(Map> filedNameToValueMap, - String oldValue, String newValue, String fieldName) { + public void getFiledNameToValueMap( + Map> filedNameToValueMap, + String oldValue, + String newValue, + String fieldName) { Map map = new HashMap<>(); map.put(oldValue, newValue); filedNameToValueMap.put(fieldName, map); } - private void rewriteDimValue(SemanticQueryResp semanticQueryResp, + private void rewriteDimValue( + SemanticQueryResp semanticQueryResp, Map> dimAndTechNameAndBizNamePair) { if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) { return; @@ -171,10 +193,12 @@ public class DimValueAspect { log.debug("start rewriteDimValue for resultList"); for (Map line : semanticQueryResp.getResultList()) { for (String bizName : line.keySet()) { - if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) { + if (dimAndTechNameAndBizNamePair.containsKey(bizName) + && Objects.nonNull(line.get(bizName))) { String techName = line.get(bizName).toString(); Map techAndBizPair = dimAndTechNameAndBizNamePair.get(bizName); - if (!CollectionUtils.isEmpty(techAndBizPair) && techAndBizPair.containsKey(techName)) { + if (!CollectionUtils.isEmpty(techAndBizPair) + && techAndBizPair.containsKey(techName)) { String bizValueName = techAndBizPair.get(techName); if (StringUtils.isNotEmpty(bizValueName)) { line.put(bizName, bizValueName); @@ -185,8 +209,9 @@ public class DimValueAspect { } } - private boolean selectDimValueMap(List columns, Map> dimAndTechNameAndBizNamePair) { + private boolean selectDimValueMap( + List columns, + Map> dimAndTechNameAndBizNamePair) { if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair) || CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) { return false; @@ -200,7 +225,8 @@ public class DimValueAspect { return false; } - private void rewriteFilter(List dimensionFilters, Map> aliasAndTechNamePair) { + private void rewriteFilter( + List dimensionFilters, Map> aliasAndTechNamePair) { for (Filter filter : dimensionFilters) { if (Objects.isNull(filter)) { continue; @@ -238,7 +264,8 @@ public class DimValueAspect { } } - private Map> getAliasAndBizNameToTechName(List dimensions) { + private Map> getAliasAndBizNameToTechName( + List dimensions) { if (CollectionUtils.isEmpty(dimensions)) { return new HashMap<>(); } @@ -256,14 +283,18 @@ public class DimValueAspect { continue; } if (StringUtils.isNotEmpty(dimValueMap.getBizName())) { - aliasAndBizNameToTechName.put(dimValueMap.getBizName(), dimValueMap.getTechName()); + aliasAndBizNameToTechName.put( + dimValueMap.getBizName(), dimValueMap.getTechName()); } if (!CollectionUtils.isEmpty(dimValueMap.getAlias())) { - dimValueMap.getAlias().stream().forEach(alias -> { - if (StringUtils.isNotEmpty(alias)) { - aliasAndBizNameToTechName.put(alias, dimValueMap.getTechName()); - } - }); + dimValueMap.getAlias().stream() + .forEach( + alias -> { + if (StringUtils.isNotEmpty(alias)) { + aliasAndBizNameToTechName.put( + alias, dimValueMap.getTechName()); + } + }); } } @@ -308,7 +339,8 @@ public class DimValueAspect { } private boolean needSkipDimension(DimensionResp dimension) { - return Objects.isNull(dimension) || StringUtils.isEmpty(dimension.getBizName()) || CollectionUtils.isEmpty( - dimension.getDimValueMaps()); + return Objects.isNull(dimension) + || StringUtils.isEmpty(dimension.getBizName()) + || CollectionUtils.isEmpty(dimension.getDimValueMaps()); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index d847b132e..5626cb28c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.aspect; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; @@ -15,6 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; @@ -22,10 +22,9 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -38,6 +37,7 @@ import org.aspectj.lang.annotation.Pointcut; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -51,22 +51,17 @@ import java.util.stream.Collectors; @Slf4j public class S2DataPermissionAspect { - @Autowired - private QueryStructUtils queryStructUtils; - @Autowired - private ModelService modelService; - @Autowired - private SchemaService schemaService; - @Autowired - private AuthService authService; + @Autowired private QueryStructUtils queryStructUtils; + @Autowired private ModelService modelService; + @Autowired private SchemaService schemaService; + @Autowired private AuthService authService; @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)") - private void s2PermissionCheck() { - } + private void s2PermissionCheck() {} @Around("s2PermissionCheck()") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { - //1. check args + // 1. check args Object[] objects = joinPoint.getArgs(); boolean needQueryData = true; SemanticQueryReq queryReq = null; @@ -88,24 +83,24 @@ public class S2DataPermissionAspect { SemanticSchemaResp semanticSchemaResp = getSemanticSchemaResp(queryReq); List modelIds = getModelIds(semanticSchemaResp); - //2. determine whether admin of the model + // 2. determine whether admin of the model if (checkModelAdmin(user, modelIds)) { return joinPoint.proceed(); } - //3. determine whether the model is visible to cur user + // 3. determine whether the model is visible to cur user checkModelVisible(user, modelIds); - //4. get permissions auth to cur user + // 4. get permissions auth to cur user AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds); - //5. check col permission + // 5. check col permission if (needQueryData) { checkColPermission(queryReq, authorizedResource, modelIds, semanticSchemaResp); } - //6. check row permission + // 6. check row permission checkRowPermission(queryReq, authorizedResource); - //7. add hint to user + // 7. add hint to user Object result = joinPoint.proceed(); if (result instanceof SemanticQueryResp) { addHint(modelIds, (SemanticQueryResp) result, authorizedResource); @@ -113,29 +108,39 @@ public class S2DataPermissionAspect { return result; } - private void checkColPermission(SemanticQueryReq semanticQueryReq, AuthorizedResourceResp authorizedResource, - List modelIds, SemanticSchemaResp semanticSchemaResp) { + private void checkColPermission( + SemanticQueryReq semanticQueryReq, + AuthorizedResourceResp authorizedResource, + List modelIds, + SemanticSchemaResp semanticSchemaResp) { // get high sensitive fields in query Set bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp); - Set sensitiveBizNamesByModel = getHighSensitiveBizNamesByModelId(semanticSchemaResp); - Set sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream() - .filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet()); + Set sensitiveBizNamesByModel = + getHighSensitiveBizNamesByModelId(semanticSchemaResp); + Set sensitiveBizNameInQuery = + bizNamesInQueryReq + .parallelStream() + .filter(sensitiveBizNamesByModel::contains) + .collect(Collectors.toSet()); - //get high sensitive field cur user has been authed - Set sensitiveBizNameUserAuthed = authorizedResource.getAuthResList() - .stream().map(AuthRes::getName).collect(Collectors.toSet()); + // get high sensitive field cur user has been authed + Set sensitiveBizNameUserAuthed = + authorizedResource.getAuthResList().stream() + .map(AuthRes::getName) + .collect(Collectors.toSet()); sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed); if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) { - Set sensitiveResNames = semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery); + Set sensitiveResNames = + semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery); List modelAdmin = modelService.getModelAdmin(modelIds.get(0)); - String message = String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请", - sensitiveResNames, modelAdmin); + String message = + String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请", sensitiveResNames, modelAdmin); throw new InvalidPermissionException(message); } } - private void checkRowPermission(SemanticQueryReq queryReq, - AuthorizedResourceResp authorizedResource) { + private void checkRowPermission( + SemanticQueryReq queryReq, AuthorizedResourceResp authorizedResource) { if (queryReq instanceof QuerySqlReq) { doRowPermission((QuerySqlReq) queryReq, authorizedResource); } @@ -144,7 +149,8 @@ public class S2DataPermissionAspect { } } - private Set getBizNameInQueryReq(SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) { + private Set getBizNameInQueryReq( + SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) { if (queryReq instanceof QuerySqlReq) { return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp); } @@ -163,10 +169,12 @@ public class S2DataPermissionAspect { private List getModelIds(SemanticSchemaResp semanticSchemaResp) { return semanticSchemaResp.getModelResps().stream() - .map(ModelResp::getId).collect(Collectors.toList()); + .map(ModelResp::getId) + .collect(Collectors.toList()); } - private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) { + private void doRowPermission( + QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); StringJoiner joiner = new StringJoiner(" OR "); List dimensionFilters = new ArrayList<>(); @@ -180,11 +188,14 @@ public class S2DataPermissionAspect { return; } - dimensionFilters.stream().forEach(filter -> { - if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) { - joiner.add(" ( " + filter + " ) "); - } - }); + dimensionFilters.stream() + .forEach( + filter -> { + if (StringUtils.isNotEmpty(filter) + && StringUtils.isNotEmpty(filter.trim())) { + joiner.add(" ( " + filter + " ) "); + } + }); try { Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); if (StringUtils.isNotEmpty(joiner.toString())) { @@ -196,10 +207,10 @@ public class S2DataPermissionAspect { } catch (JSQLParserException jsqlParserException) { log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); } - } - private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) { + private void doRowPermission( + QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); StringJoiner joiner = new StringJoiner(" OR "); List dimensionFilters = new ArrayList<>(); @@ -213,37 +224,45 @@ public class S2DataPermissionAspect { return; } - dimensionFilters.stream().forEach(filter -> { - if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) { - joiner.add(" ( " + filter + " ) "); - } - }); + dimensionFilters.stream() + .forEach( + filter -> { + if (StringUtils.isNotEmpty(filter) + && StringUtils.isNotEmpty(filter.trim())) { + joiner.add(" ( " + filter + " ) "); + } + }); if (StringUtils.isNotEmpty(joiner.toString())) { log.info("before doRowPermission, queryStructReq:{}", queryStructReq); Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString()); - List filters = Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>() - : queryStructReq.getOriginalFilter(); + List filters = + Objects.isNull(queryStructReq.getOriginalFilter()) + ? new ArrayList<>() + : queryStructReq.getOriginalFilter(); filters.add(filter); queryStructReq.setDimensionFilters(filters); log.info("after doRowPermission, queryStructReq:{}", queryStructReq); } - } public boolean checkModelAdmin(User user, List modelIds) { - List modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); + List modelListAdmin = + modelService.getModelListWithAuth(user, null, AuthType.ADMIN); if (CollectionUtils.isEmpty(modelListAdmin)) { return false; } else { - Set modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet()); + Set modelAdmins = + modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet()); return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds); } } public void checkModelVisible(User user, List modelIds) { - List modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE) - .stream().map(ModelResp::getId).collect(Collectors.toList()); + List modelListVisible = + modelService.getModelListWithAuth(user, null, AuthType.VISIBLE).stream() + .map(ModelResp::getId) + .collect(Collectors.toList()); List modelIdCopied = new ArrayList<>(modelIds); modelIdCopied.removeAll(modelListVisible); if (!CollectionUtils.isEmpty(modelIdCopied)) { @@ -254,7 +273,9 @@ public class S2DataPermissionAspect { if (modelResp == null) { throw new InvalidArgumentException("查询的模型不存在"); } - String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins()); + String message = + String.format( + "您没有模型[%s]权限，请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins()); throw new InvalidPermissionException(message); } } @@ -262,13 +283,21 @@ public class S2DataPermissionAspect { public Set getHighSensitiveBizNamesByModelId(SemanticSchemaResp semanticSchemaResp) { Set highSensitiveCols = new HashSet<>(); if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) { - semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp -> - SensitiveLevelEnum.HIGH.getCode().equals(dimSchemaResp.getSensitiveLevel())) + semanticSchemaResp.getDimensions().stream() + .filter( + dimSchemaResp -> + SensitiveLevelEnum.HIGH + .getCode() + .equals(dimSchemaResp.getSensitiveLevel())) .forEach(dim -> highSensitiveCols.add(dim.getBizName())); } if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) { - semanticSchemaResp.getMetrics().stream().filter(metricSchemaResp -> - SensitiveLevelEnum.HIGH.getCode().equals(metricSchemaResp.getSensitiveLevel())) + semanticSchemaResp.getMetrics().stream() + .filter( + metricSchemaResp -> + SensitiveLevelEnum.HIGH + .getCode() + .equals(metricSchemaResp.getSensitiveLevel())) .forEach(metric -> highSensitiveCols.add(metric.getBizName())); } return highSensitiveCols; @@ -278,8 +307,11 @@ public class S2DataPermissionAspect { QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); queryAuthResReq.setModelIds(modelIds); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); - log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", - user.getName(), modelIds, authorizedResource); + log.info( + "user:{}, domainId:{}, after queryAuthorizedResources:{}", + user.getName(), + modelIds, + authorizedResource); return authorizedResource; } @@ -288,8 +320,10 @@ public class S2DataPermissionAspect { return authService.queryAuthorizedResources(queryAuthResReq, user); } - public void addHint(List modelIds, SemanticQueryResp queryResultWithColumns, - AuthorizedResourceResp authorizedResource) { + public void addHint( + List modelIds, + SemanticQueryResp queryResultWithColumns, + AuthorizedResourceResp authorizedResource) { List filters = authorizedResource.getFilters(); if (CollectionUtils.isEmpty(filters)) { return; @@ -300,17 +334,22 @@ public class S2DataPermissionAspect { ModelResp modelResp = modelService.getModel(modelIds.get(0)); List exprList = new ArrayList<>(); List descList = new ArrayList<>(); - filters.stream().forEach(filter -> { - if (StringUtils.isNotEmpty(filter.getDescription())) { - descList.add(filter.getDescription()); - } - exprList.add(filter.getExpressions().toString()); - }); + filters.stream() + .forEach( + filter -> { + if (StringUtils.isNotEmpty(filter.getDescription())) { + descList.add(filter.getDescription()); + } + exprList.add(filter.getExpressions().toString()); + }); String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s"; - String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins); + String message = + String.format( + promptInfo, + CollectionUtils.isEmpty(descList) ? exprList : descList, + admins); queryResultWithColumns.setQueryAuthorization( new QueryAuthorization(modelResp.getName(), exprList, descList, message)); } } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/ChatQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/ChatQueryApiController.java index c63b36c3d..08533b9bb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/ChatQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/ChatQueryApiController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.facade.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; @@ -14,48 +17,51 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; @RestController @RequestMapping("/api/semantic/query") @Slf4j public class ChatQueryApiController { - @Autowired - private ChatLayerService chatLayerService; + @Autowired private ChatLayerService chatLayerService; - @Autowired - private SemanticLayerService semanticLayerService; + @Autowired private SemanticLayerService semanticLayerService; @PostMapping("/chat/search") - public Object search(@RequestBody QueryNLReq queryNLReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Object search( + @RequestBody QueryNLReq queryNLReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { queryNLReq.setUser(UserHolder.findUser(request, response)); return chatLayerService.retrieve(queryNLReq); } @PostMapping("/chat/map") - public Object map(@RequestBody QueryNLReq queryNLReq, - HttpServletRequest request, - HttpServletResponse response) { + public Object map( + @RequestBody QueryNLReq queryNLReq, + HttpServletRequest request, + HttpServletResponse response) { queryNLReq.setUser(UserHolder.findUser(request, response)); return chatLayerService.performMapping(queryNLReq); } @PostMapping("/chat/parse") - public Object parse(@RequestBody QueryNLReq queryNLReq, + public Object parse( + @RequestBody QueryNLReq queryNLReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { queryNLReq.setUser(UserHolder.findUser(request, response)); return chatLayerService.performParsing(queryNLReq); } @PostMapping("/chat") - public Object queryByNL(@RequestBody QueryNLReq queryNLReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Object queryByNL( + @RequestBody QueryNLReq queryNLReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); ParseResp parseResp = chatLayerService.performParsing(queryNLReq); if (parseResp.getState().equals(ParseResp.ParseState.COMPLETED)) { @@ -66,7 +72,7 @@ public class ChatQueryApiController { return semanticLayerService.queryByReq(sqlReq, user); } - throw new RuntimeException("Failed to parse natural language query: " + queryNLReq.getQueryText()); + throw new RuntimeException( + "Failed to parse natural language query: " + queryNLReq.getQueryText()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/DataSetQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/DataSetQueryApiController.java index b35d62653..e4b4f325b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/DataSetQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/DataSetQueryApiController.java @@ -1,11 +1,14 @@ package com.tencent.supersonic.headless.server.facade.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; +import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -13,26 +16,22 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - @RestController @RequestMapping("/api/semantic/query") @Slf4j public class DataSetQueryApiController { - @Autowired - private DataSetService dataSetService; - @Autowired - private SemanticLayerService semanticLayerService; + @Autowired private DataSetService dataSetService; + @Autowired private SemanticLayerService semanticLayerService; @PostMapping("/dataSet") - public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq, + public Object queryByDataSet( + @RequestBody QueryDataSetReq queryDataSetReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq); return semanticLayerService.queryByReq(queryReq, user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetaDiscoveryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetaDiscoveryApiController.java index c160d5e02..1384a5973 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetaDiscoveryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetaDiscoveryApiController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.facade.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; @@ -11,23 +14,21 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - @RestController @RequestMapping("/api/semantic/meta") @Slf4j public class MetaDiscoveryApiController { - @Autowired - private ChatLayerService chatLayerService; + @Autowired private ChatLayerService chatLayerService; @PostMapping("map") - public Object map(@RequestBody QueryMapReq queryMapReq, - HttpServletRequest request, HttpServletResponse response) throws Exception { + public Object map( + @RequestBody QueryMapReq queryMapReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); queryMapReq.setUser(user); return chatLayerService.map(queryMapReq); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetricQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetricQueryApiController.java index 03146c68b..d95414572 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetricQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetricQueryApiController.java @@ -1,14 +1,17 @@ package com.tencent.supersonic.headless.server.facade.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.service.DownloadService; import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -16,46 +19,45 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - @RestController @RequestMapping("/api/semantic/query") @Slf4j public class MetricQueryApiController { - @Autowired - private SemanticLayerService semanticLayerService; + @Autowired private SemanticLayerService semanticLayerService; - @Autowired - private MetricService metricService; + @Autowired private MetricService metricService; - @Autowired - private DownloadService downloadService; + @Autowired private DownloadService downloadService; @PostMapping("/metric") - public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq, + public Object queryByMetric( + @RequestBody QueryMetricReq queryMetricReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); QueryStructReq queryStructReq = metricService.convert(queryMetricReq); return semanticLayerService.queryByReq(queryStructReq.convert(true), user); } @PostMapping("/download/metric") - public void downloadMetric(@RequestBody DownloadMetricReq downloadMetricReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public void downloadMetric( + @RequestBody DownloadMetricReq downloadMetricReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); downloadService.downloadByStruct(downloadMetricReq, user, response); } @PostMapping("/downloadBatch/metric") - public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public void downloadBatch( + @RequestBody BatchDownloadReq batchDownloadReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); downloadService.batchDownload(batchDownloadReq, user, response); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java index ccf6e4417..7dd387445 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.facade.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.util.StringUtil; @@ -17,8 +20,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -29,16 +30,16 @@ import java.util.stream.Collectors; @Slf4j public class SqlQueryApiController { - @Autowired - private SemanticLayerService semanticLayerService; + @Autowired private SemanticLayerService semanticLayerService; - @Autowired - private ChatLayerService chatLayerService; + @Autowired private ChatLayerService chatLayerService; @PostMapping("/sql") - public Object queryBySql(@RequestBody QuerySqlReq querySqlReq, + public Object queryBySql( + @RequestBody QuerySqlReq querySqlReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); String sql = querySqlReq.getSql(); querySqlReq.setSql(StringUtil.replaceBackticks(sql)); @@ -47,48 +48,68 @@ public class SqlQueryApiController { } @PostMapping("/sqls") - public Object queryBySqls(@RequestBody QuerySqlsReq querySqlsReq, + public Object queryBySqls( + @RequestBody QuerySqlsReq querySqlsReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); - List semanticQueryReqs = querySqlsReq.getSqls() - .stream().map(sql -> { - QuerySqlReq querySqlReq = new QuerySqlReq(); - BeanUtils.copyProperties(querySqlsReq, querySqlReq); - querySqlReq.setSql(StringUtil.replaceBackticks(sql)); - chatLayerService.correct(querySqlReq, user); - return querySqlReq; - }).collect(Collectors.toList()); + List semanticQueryReqs = + querySqlsReq.getSqls().stream() + .map( + sql -> { + QuerySqlReq querySqlReq = new QuerySqlReq(); + BeanUtils.copyProperties(querySqlsReq, querySqlReq); + querySqlReq.setSql(StringUtil.replaceBackticks(sql)); + chatLayerService.correct(querySqlReq, user); + return querySqlReq; + }) + .collect(Collectors.toList()); - List> futures = semanticQueryReqs.stream() - .map(querySqlReq -> CompletableFuture.supplyAsync(() -> { - try { - return semanticLayerService.queryByReq(querySqlReq, user); - } catch (Exception e) { - log.error("querySqlReq:{},queryByReq error:", querySqlReq, e); - return new SemanticQueryResp(); - } - })).collect(Collectors.toList()); + List> futures = + semanticQueryReqs.stream() + .map( + querySqlReq -> + CompletableFuture.supplyAsync( + () -> { + try { + return semanticLayerService.queryByReq( + querySqlReq, user); + } catch (Exception e) { + log.error( + "querySqlReq:{},queryByReq error:", + querySqlReq, + e); + return new SemanticQueryResp(); + } + })) + .collect(Collectors.toList()); return futures.stream().map(CompletableFuture::join).collect(Collectors.toList()); } @PostMapping("/sqlsWithException") - public Object queryBySqlsWithException(@RequestBody QuerySqlsReq querySqlsReq, + public Object queryBySqlsWithException( + @RequestBody QuerySqlsReq querySqlsReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); - List semanticQueryReqs = querySqlsReq.getSqls() - .stream().map(sql -> { - QuerySqlReq querySqlReq = new QuerySqlReq(); - BeanUtils.copyProperties(querySqlsReq, querySqlReq); - querySqlReq.setSql(StringUtil.replaceBackticks(sql)); - chatLayerService.correct(querySqlReq, user); - return querySqlReq; - }).collect(Collectors.toList()); + List semanticQueryReqs = + querySqlsReq.getSqls().stream() + .map( + sql -> { + QuerySqlReq querySqlReq = new QuerySqlReq(); + BeanUtils.copyProperties(querySqlsReq, querySqlReq); + querySqlReq.setSql(StringUtil.replaceBackticks(sql)); + chatLayerService.correct(querySqlReq, user); + return querySqlReq; + }) + .collect(Collectors.toList()); List semanticQueryRespList = new ArrayList<>(); try { for (SemanticQueryReq semanticQueryReq : semanticQueryReqs) { - SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(semanticQueryReq, user); + SemanticQueryResp semanticQueryResp = + semanticLayerService.queryByReq(semanticQueryReq, user); semanticQueryRespList.add(semanticQueryResp); } } catch (Exception e) { @@ -98,13 +119,14 @@ public class SqlQueryApiController { } @PostMapping("/validate") - public Object validate(@RequestBody QuerySqlReq querySqlReq, + public Object validate( + @RequestBody QuerySqlReq querySqlReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); String sql = querySqlReq.getSql(); querySqlReq.setSql(StringUtil.replaceBackticks(sql)); return chatLayerService.validate(querySqlReq, user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/TagQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/TagQueryApiController.java index 99a0d871e..67ce37d65 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/TagQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/TagQueryApiController.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.headless.server.facade.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -19,15 +19,15 @@ import org.springframework.web.bind.annotation.RestController; @Slf4j public class TagQueryApiController { - @Autowired - private SemanticLayerService semanticLayerService; + @Autowired private SemanticLayerService semanticLayerService; @PostMapping("/tag") - public Object queryByTag(@RequestBody QueryStructReq queryStructReq, + public Object queryByTag( + @RequestBody QueryStructReq queryStructReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return semanticLayerService.queryByReq(queryStructReq.convert(), user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/ChatLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/ChatLayerService.java index 82c7e2b7c..ec46b14c3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/ChatLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/ChatLayerService.java @@ -12,9 +12,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SearchResult; import java.util.List; -/** - * This interface adds natural language support to the semantic layer. - */ +/** This interface adds natural language support to the semantic layer. */ public interface ChatLayerService { MapResp performMapping(QueryNLReq queryNLReq); @@ -29,4 +27,3 @@ public interface ChatLayerService { SqlEvaluation validate(QuerySqlReq querySqlReq, User user); } - diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java index 2a5281c76..87f82a3e8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java @@ -15,9 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp; import java.util.List; -/** - * This interface abstracts functionalities provided by a semantic layer. - */ +/** This interface abstracts functionalities provided by a semantic layer. */ public interface SemanticLayerService { DataSetSchema getDataSetSchema(Long id); @@ -35,5 +33,4 @@ public interface SemanticLayerService { List getDimensions(MetaFilter metaFilter); List getMetrics(MetaFilter metaFilter); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java index 0ff7b7857..8ea16788b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java @@ -5,6 +5,7 @@ import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; @@ -28,12 +29,11 @@ import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector; import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector; import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.RetrieveService; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.utils.ChatWorkflowEngine; import com.tencent.supersonic.headless.server.utils.ComponentFactory; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -53,22 +53,20 @@ import java.util.stream.Collectors; @Service @Slf4j public class S2ChatLayerService implements ChatLayerService { - @Autowired - private SchemaService schemaService; - @Autowired - private DataSetService dataSetService; - @Autowired - private RetrieveService retrieveService; - @Autowired - private ChatWorkflowEngine chatWorkflowEngine; + @Autowired private SchemaService schemaService; + @Autowired private DataSetService dataSetService; + @Autowired private RetrieveService retrieveService; + @Autowired private ChatWorkflowEngine chatWorkflowEngine; @Override public MapResp performMapping(QueryNLReq queryNLReq) { MapResp mapResp = new MapResp(queryNLReq.getQueryText()); ChatQueryContext queryCtx = buildChatQueryContext(queryNLReq); - ComponentFactory.getSchemaMappers().forEach(mapper -> { - mapper.map(queryCtx); - }); + ComponentFactory.getSchemaMappers() + .forEach( + mapper -> { + mapper.map(queryCtx); + }); mapResp.setMapInfo(queryCtx.getMapInfo()); return mapResp; } @@ -77,7 +75,8 @@ public class S2ChatLayerService implements ChatLayerService { public MapInfoResp map(QueryMapReq queryMapReq) { QueryNLReq queryNLReq = new QueryNLReq(); BeanUtils.copyProperties(queryMapReq, queryNLReq); - List dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser()); + List dataSets = + dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser()); Set dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet()); queryNLReq.setDataSetIds(dataSetIds); @@ -97,16 +96,17 @@ public class S2ChatLayerService implements ChatLayerService { private ChatQueryContext buildChatQueryContext(QueryNLReq queryNLReq) { SemanticSchema semanticSchema = schemaService.getSemanticSchema(queryNLReq.getDataSetIds()); Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(); - ChatQueryContext queryCtx = ChatQueryContext.builder() - .queryFilters(queryNLReq.getQueryFilters()) - .semanticSchema(semanticSchema) - .candidateQueries(new ArrayList<>()) - .mapInfo(new SchemaMapInfo()) - .modelIdToDataSetIds(modelIdToDataSetIds) - .text2SQLType(queryNLReq.getText2SQLType()) - .mapModeEnum(queryNLReq.getMapModeEnum()) - .dataSetIds(queryNLReq.getDataSetIds()) - .build(); + ChatQueryContext queryCtx = + ChatQueryContext.builder() + .queryFilters(queryNLReq.getQueryFilters()) + .semanticSchema(semanticSchema) + .candidateQueries(new ArrayList<>()) + .mapInfo(new SchemaMapInfo()) + .modelIdToDataSetIds(modelIdToDataSetIds) + .text2SQLType(queryNLReq.getText2SQLType()) + .mapModeEnum(queryNLReq.getMapModeEnum()) + .dataSetIds(queryNLReq.getDataSetIds()) + .build(); BeanUtils.copyProperties(queryNLReq, queryCtx); return queryCtx; } @@ -129,8 +129,8 @@ public class S2ChatLayerService implements ChatLayerService { private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) { ChatQueryContext queryCtx = new ChatQueryContext(); - SemanticSchema semanticSchema = schemaService.getSemanticSchema( - Sets.newHashSet(querySqlReq.getDataSetId())); + SemanticSchema semanticSchema = + schemaService.getSemanticSchema(Sets.newHashSet(querySqlReq.getDataSetId())); queryCtx.setSemanticSchema(semanticSchema); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); SqlInfo sqlInfo = new SqlInfo(); @@ -146,11 +146,14 @@ public class S2ChatLayerService implements ChatLayerService { SchemaElement dataSet = semanticSchema.getDataSet(dataSetId); semanticParseInfo.setDataSet(dataSet); - ComponentFactory.getSemanticCorrectors().forEach(corrector -> { - if (!(corrector instanceof GrammarCorrector || (corrector instanceof SchemaCorrector))) { - corrector.correct(queryCtx, semanticParseInfo); - } - }); + ComponentFactory.getSemanticCorrectors() + .forEach( + corrector -> { + if (!(corrector instanceof GrammarCorrector + || (corrector instanceof SchemaCorrector))) { + corrector.correct(queryCtx, semanticParseInfo); + } + }); log.info("chatQueryServiceImpl correct:{}", sqlInfo.getCorrectedS2SQL()); return semanticParseInfo; } @@ -164,16 +167,15 @@ public class S2ChatLayerService implements ChatLayerService { MetaFilter metaFilter = new MetaFilter(); metaFilter.setIds(new ArrayList<>(dataSetIds)); List dataSetList = dataSetService.getDataSetList(metaFilter); - Map dataSetMap = dataSetList.stream() - .collect(Collectors.toMap(DataSetResp::getId, d -> d)); + Map dataSetMap = + dataSetList.stream().collect(Collectors.toMap(DataSetResp::getId, d -> d)); mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN)); mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap)); return mapInfoResp; } - private Map getDataSetInfo(SchemaMapInfo mapInfo, - Map dataSetMap, - Integer topN) { + private Map getDataSetInfo( + SchemaMapInfo mapInfo, Map dataSetMap, Integer topN) { Map map = new HashMap<>(); Map> mapFields = getMapFields(mapInfo, dataSetMap); Map> topFields = getTopFields(topN, mapInfo, dataSetMap); @@ -195,14 +197,18 @@ public class S2ChatLayerService implements ChatLayerService { return map; } - private Map> getMapFields(SchemaMapInfo mapInfo, - Map dataSetMap) { + private Map> getMapFields( + SchemaMapInfo mapInfo, Map dataSetMap) { Map> result = new HashMap<>(); - for (Map.Entry> entry : mapInfo.getDataSetElementMatches().entrySet()) { - List values = entry.getValue().stream() - .filter(schemaElementMatch -> - !SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); + for (Map.Entry> entry : + mapInfo.getDataSetElementMatches().entrySet()) { + List values = + entry.getValue().stream() + .filter( + schemaElementMatch -> + !SchemaElementType.TERM.equals( + schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) { result.put(entry.getKey(), values); } @@ -210,15 +216,15 @@ public class S2ChatLayerService implements ChatLayerService { return result; } - private Map> getTopFields(Integer topN, - SchemaMapInfo mapInfo, - Map dataSetMap) { + private Map> getTopFields( + Integer topN, SchemaMapInfo mapInfo, Map dataSetMap) { Map> result = new HashMap<>(); if (0 == topN) { return result; } SemanticSchema semanticSchema = schemaService.getSemanticSchema(); - for (Map.Entry> entry : mapInfo.getDataSetElementMatches().entrySet()) { + for (Map.Entry> entry : + mapInfo.getDataSetElementMatches().entrySet()) { Long dataSetId = entry.getKey(); List values = entry.getValue(); DataSetResp dataSetResp = dataSetMap.get(dataSetId); @@ -226,18 +232,24 @@ public class S2ChatLayerService implements ChatLayerService { continue; } String dataSetName = dataSetResp.getName(); - //topN dimensions - Set dimensions = semanticSchema.getDimensions(dataSetId) - .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet()); + // topN dimensions + Set dimensions = + semanticSchema.getDimensions(dataSetId).stream() + .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) + .limit(topN - 1) + .map(mergeFunction()) + .collect(Collectors.toSet()); SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName); dimensions.add(timeDimensionMatch); - //topN metrics - Set metrics = semanticSchema.getMetrics(dataSetId) - .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .limit(topN).map(mergeFunction()).collect(Collectors.toSet()); + // topN metrics + Set metrics = + semanticSchema.getMetrics(dataSetId).stream() + .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) + .limit(topN) + .map(mergeFunction()) + .collect(Collectors.toSet()); dimensions.addAll(metrics); result.put(dataSetId, new ArrayList<>(dimensions)); @@ -245,44 +257,64 @@ public class S2ChatLayerService implements ChatLayerService { return result; } - private Map> getTerms(SchemaMapInfo mapInfo, - Map dataSetNameMap) { + private Map> getTerms( + SchemaMapInfo mapInfo, Map dataSetNameMap) { Map> termMap = new HashMap<>(); - Map> dataSetElementMatches = mapInfo.getDataSetElementMatches(); + Map> dataSetElementMatches = + mapInfo.getDataSetElementMatches(); for (Map.Entry> entry : dataSetElementMatches.entrySet()) { DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey()); if (dataSetResp == null) { continue; } - List terms = entry.getValue().stream().filter(schemaElementMatch - -> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); + List terms = + entry.getValue().stream() + .filter( + schemaElementMatch -> + SchemaElementType.TERM.equals( + schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); termMap.put(dataSetResp.getName(), terms); } return termMap; } - /*** - * get time dimension SchemaElementMatch + /** + * * get time dimension SchemaElementMatch + * * @param dataSetId * @param dataSetName * @return */ private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) { - SchemaElement element = SchemaElement.builder().dataSetId(dataSetId).dataSetName(dataSetName) - .type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build(); + SchemaElement element = + SchemaElement.builder() + .dataSetId(dataSetId) + .dataSetName(dataSetName) + .type(SchemaElementType.DIMENSION) + .bizName(TimeDimensionEnum.DAY.getName()) + .build(); - SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element) - .detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName()) - .similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build(); + SchemaElementMatch timeDimensionMatch = + SchemaElementMatch.builder() + .element(element) + .detectWord(TimeDimensionEnum.DAY.getChName()) + .word(TimeDimensionEnum.DAY.getChName()) + .similarity(1L) + .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) + .build(); return timeDimensionMatch; } private Function mergeFunction() { - return schemaElement -> SchemaElementMatch.builder().element(schemaElement) - .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1) - .detectWord(schemaElement.getName()).build(); + return schemaElement -> + SchemaElementMatch.builder() + .element(schemaElement) + .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) + .word(schemaElement.getName()) + .similarity(1) + .detectWord(schemaElement.getName()) + .build(); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index 301b633c9..de6badf21 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -52,14 +52,14 @@ import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.server.annotation.S2DataPermission; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; -import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker; -import com.tencent.supersonic.headless.server.utils.QueryReqConverter; -import com.tencent.supersonic.headless.server.utils.QueryUtils; -import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker; +import com.tencent.supersonic.headless.server.utils.QueryReqConverter; +import com.tencent.supersonic.headless.server.utils.QueryUtils; +import com.tencent.supersonic.headless.server.utils.StatUtils; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -77,7 +77,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; - @Service @Slf4j public class S2SemanticLayerService implements SemanticLayerService { @@ -144,10 +143,10 @@ public class S2SemanticLayerService implements SemanticLayerService { TaskStatusEnum state = TaskStatusEnum.SUCCESS; log.info("[queryReq:{}]", queryReq); try { - //1.initStatInfo + // 1.initStatInfo statUtils.initStatInfo(queryReq, user); - //2.query from cache + // 2.query from cache String cacheKey = queryCache.getCacheKey(queryReq); Object query = queryCache.query(queryReq, cacheKey); @@ -159,7 +158,7 @@ public class S2SemanticLayerService implements SemanticLayerService { } StatUtils.get().setUseResultCache(false); - //3 query + // 3 query QueryStatement queryStatement = buildQueryStatement(queryReq, user); SemanticQueryResp queryResp = null; @@ -172,11 +171,12 @@ public class S2SemanticLayerService implements SemanticLayerService { for (QueryExecutor queryExecutor : queryExecutors) { if (queryExecutor.accept(queryStatement)) { queryResp = queryExecutor.execute(queryStatement); - queryUtils.populateQueryColumns(queryResp, queryStatement.getSemanticSchemaResp()); + queryUtils.populateQueryColumns( + queryResp, queryStatement.getSemanticSchemaResp()); } } - //4 reset cache and set stateInfo + // 4 reset cache and set stateInfo Boolean setCacheSuccess = queryCache.put(cacheKey, queryResp); if (setCacheSuccess) { // if result is not null, update cache data @@ -218,7 +218,8 @@ public class S2SemanticLayerService implements SemanticLayerService { return semanticQueryResp; } - private List getDimensionValuesFromDict(DimensionValueReq dimensionValueReq, Set dataSetIds) { + private List getDimensionValuesFromDict( + DimensionValueReq dimensionValueReq, Set dataSetIds) { if (StringUtils.isBlank(dimensionValueReq.getValue())) { return SearchService.getDimensionValue(dimensionValueReq); } @@ -226,20 +227,28 @@ public class S2SemanticLayerService implements SemanticLayerService { Map> modelIdToDataSetIds = new HashMap<>(); modelIdToDataSetIds.put(dimensionValueReq.getModelId(), new ArrayList<>(dataSetIds)); - List hanlpMapResultList = knowledgeBaseService.prefixSearch( - dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds); + List hanlpMapResultList = + knowledgeBaseService.prefixSearch( + dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds); HanlpHelper.transLetterOriginal(hanlpMapResultList); return hanlpMapResultList.stream() - .filter(o -> o.getNatures().stream() - .map(NatureHelper::getElementID) - .anyMatch(elementID -> dimensionValueReq.getElementID().equals(elementID))) + .filter( + o -> + o.getNatures().stream() + .map(NatureHelper::getElementID) + .anyMatch( + elementID -> + dimensionValueReq + .getElementID() + .equals(elementID))) .map(MapResult::getName) .collect(Collectors.toList()); } - private SemanticQueryResp getDimensionValuesFromDb(DimensionValueReq dimensionValueReq, User user) { + private SemanticQueryResp getDimensionValuesFromDb( + DimensionValueReq dimensionValueReq, User user) { QuerySqlReq querySqlReq = buildQuerySqlReq(dimensionValueReq); return queryByReq(querySqlReq, user); } @@ -256,14 +265,15 @@ public class S2SemanticLayerService implements SemanticLayerService { return columns; } - private List> createResultList(DimensionValueReq dimensionValueReq, - List dimensionValues) { + private List> createResultList( + DimensionValueReq dimensionValueReq, List dimensionValues) { return dimensionValues.stream() - .map(value -> { - Map map = new HashMap<>(); - map.put(dimensionValueReq.getBizName(), value); - return map; - }) + .map( + value -> { + Map map = new HashMap<>(); + map.put(dimensionValueReq.getBizName(), value); + return map; + }) .collect(Collectors.toList()); } @@ -278,10 +288,12 @@ public class S2SemanticLayerService implements SemanticLayerService { return dimensionResp; } - public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) { + public EntityInfo getEntityInfo( + SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) { if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) { EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema); - if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) { + if (parseInfo.getDimensionFilters().size() <= 0 + || entityInfo.getDataSetInfo() == null) { entityInfo.setMetrics(null); entityInfo.setDimensions(null); return entityInfo; @@ -290,8 +302,9 @@ public class S2SemanticLayerService implements SemanticLayerService { if (StringUtils.isNotBlank(primaryKey)) { String entityId = ""; for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { - if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName() - .equals(primaryKey)) { + if (chatFilter != null + && chatFilter.getBizName() != null + && chatFilter.getBizName().equals(primaryKey)) { if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { entityId = chatFilter.getValue().toString(); } @@ -344,8 +357,9 @@ public class S2SemanticLayerService implements SemanticLayerService { return metrics; } - private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception { - //If dataSetId or DataSetName is empty, parse dataSetId from the SQL + private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) + throws Exception { + // If dataSetId or DataSetName is empty, parse dataSetId from the SQL if (querySqlReq.needGetDataSetId()) { Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user); querySqlReq.setDataSetId(dataSetId); @@ -360,7 +374,8 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryStatement; } - private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception { + private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) + throws Exception { QueryStatement queryStatement = null; if (semanticQueryReq instanceof QuerySqlReq) { queryStatement = buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user); @@ -369,9 +384,11 @@ public class S2SemanticLayerService implements SemanticLayerService { queryStatement = buildStructQueryStatement((QueryStructReq) semanticQueryReq); } if (semanticQueryReq instanceof QueryMultiStructReq) { - queryStatement = buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user); + queryStatement = + buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user); } - if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo()) + if (Objects.nonNull(queryStatement) + && Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) { queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL()); queryStatement.setDataSetId(semanticQueryReq.getDataSetId()); @@ -395,8 +412,8 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryStatement; } - private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq, User user) - throws Exception { + private QueryStatement buildMultiStructQueryStatement( + QueryMultiStructReq queryMultiStructReq, User user) throws Exception { List sqlParsers = new ArrayList<>(); for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { QueryStatement queryStatement = buildQueryStatement(queryStructReq, user); @@ -420,20 +437,34 @@ public class S2SemanticLayerService implements SemanticLayerService { private QuerySqlReq buildQuerySqlReq(DimensionValueReq queryDimValueReq) { QuerySqlReq querySqlReq = new QuerySqlReq(); - List modelResps = schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); - DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getBizName(), - queryDimValueReq.getModelId()); + List modelResps = + schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); + DimensionResp dimensionResp = + schemaService.getDimension( + queryDimValueReq.getBizName(), queryDimValueReq.getModelId()); ModelResp modelResp = modelResps.get(0); - String sql = String.format("select distinct %s from %s where 1=1", - dimensionResp.getName(), modelResp.getName()); + String sql = + String.format( + "select distinct %s from %s where 1=1", + dimensionResp.getName(), modelResp.getName()); List timeDims = modelResp.getTimeDimension(); if (CollectionUtils.isNotEmpty(timeDims)) { - sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(), - queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(), - queryDimValueReq.getDateInfo().getEndDate()); + sql = + String.format( + "%s and %s >= '%s' and %s <= '%s'", + sql, + TimeDimensionEnum.DAY.getName(), + queryDimValueReq.getDateInfo().getStartDate(), + TimeDimensionEnum.DAY.getName(), + queryDimValueReq.getDateInfo().getEndDate()); } if (StringUtils.isNotBlank(queryDimValueReq.getValue())) { - sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%" + queryDimValueReq.getValue() + "%'"; + sql += + " AND " + + queryDimValueReq.getBizName() + + " LIKE '%" + + queryDimValueReq.getValue() + + "%'"; } querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); querySqlReq.setSql(sql); @@ -441,8 +472,8 @@ public class S2SemanticLayerService implements SemanticLayerService { } private void queryPreCheck(QueryStatement queryStatement) { - //Check whether the dimensions of the metric drill-down are correct temporarily, - //add the abstraction of a validator later. + // Check whether the dimensions of the metric drill-down are correct temporarily, + // add the abstraction of a validator later. metricDrillDownChecker.checkQuery(queryStatement); } @@ -466,28 +497,48 @@ public class S2SemanticLayerService implements SemanticLayerService { if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) { return entityInfo; } - List dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> { - SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id); - if (element == null) { - return null; - } - return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); - }).filter(Objects::nonNull).collect(Collectors.toList()); - List metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> { - SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id); - if (element == null) { - return null; - } - return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); - }).filter(Objects::nonNull).collect(Collectors.toList()); + List dimensions = + tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() + .map( + id -> { + SchemaElement element = + dataSetSchema.getElement( + SchemaElementType.DIMENSION, id); + if (element == null) { + return null; + } + return new DataInfo( + element.getId().intValue(), + element.getName(), + element.getBizName(), + null); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + List metrics = + tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() + .map( + id -> { + SchemaElement element = + dataSetSchema.getElement(SchemaElementType.METRIC, id); + if (element == null) { + return null; + } + return new DataInfo( + element.getId().intValue(), + element.getName(), + element.getBizName(), + null); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); entityInfo.setDimensions(dimensions); entityInfo.setMetrics(metrics); return entityInfo; } - private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) { + private void fillEntityInfoValue( + EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) { SemanticQueryResp queryResultWithColumns = getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user); if (queryResultWithColumns != null) { @@ -498,17 +549,19 @@ public class S2SemanticLayerService implements SemanticLayerService { if (entry.getValue() == null || entryKey == null) { continue; } - entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName())) + entityInfo.getDimensions().stream() + .filter(i -> entryKey.equals(i.getBizName())) .forEach(i -> i.setValue(entry.getValue().toString())); - entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName())) + entityInfo.getMetrics().stream() + .filter(i -> entryKey.equals(i.getBizName())) .forEach(i -> i.setValue(entry.getValue().toString())); } } } } - private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, - DataSetSchema dataSetSchema, User user) { + private SemanticQueryResp getQueryResultWithSchemaResp( + EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) { SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setDataSet(dataSetSchema.getDataSet()); semanticParseInfo.setQueryType(QueryType.DETAIL); @@ -532,7 +585,7 @@ public class S2SemanticLayerService implements SemanticLayerService { semanticParseInfo.setDateInfo(dateInfo); } - //add filter + // add filter QueryFilter chatFilter = getQueryFilter(entityInfo); Set chatFilters = Sets.newHashSet(); chatFilters.add(chatFilter); @@ -568,5 +621,4 @@ public class S2SemanticLayerService implements SemanticLayerService { private String getEntityPrimaryName(EntityInfo entityInfo) { return entityInfo.getDataSetInfo().getPrimaryKey(); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java index 6404b2eef..7269e5c1d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java @@ -21,11 +21,9 @@ import java.util.List; @Slf4j public class MetaEmbeddingListener implements ApplicationListener { - @Autowired - private EmbeddingConfig embeddingConfig; + @Autowired private EmbeddingConfig embeddingConfig; - @Autowired - private EmbeddingService embeddingService; + @Autowired private EmbeddingService embeddingService; @Value("${s2.embedding.operation.sleep.time:3000}") private Integer embeddingOperationSleepTime; @@ -59,5 +57,4 @@ public class MetaEmbeddingListener implements ApplicationListener { log.error("", e); } } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java index 6561821e0..f2a7c3c04 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java @@ -22,24 +22,28 @@ public class SchemaDictUpdateListener implements ApplicationListener if (CollectionUtils.isEmpty(dataEvent.getDataItems())) { return; } - dataEvent.getDataItems().forEach(dataItem -> { - DictWord dictWord = new DictWord(); - dictWord.setWord(dataItem.getName()); - String sign = DictWordType.NATURE_SPILT; - String suffixNature = DictWordType.getSuffixNature(dataItem.getType()); - String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature; - String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; - dictWord.setNature(nature); - dictWord.setNatureWithFrequency(natureWithFrequency); - if (EventType.ADD.equals(dataEvent.getEventType())) { - HanlpHelper.addToCustomDictionary(dictWord); - } else if (EventType.DELETE.equals(dataEvent.getEventType())) { - HanlpHelper.removeFromCustomDictionary(dictWord); - } else if (EventType.UPDATE.equals(dataEvent.getEventType())) { - HanlpHelper.removeFromCustomDictionary(dictWord); - dictWord.setWord(dataItem.getNewName()); - HanlpHelper.addToCustomDictionary(dictWord); - } - }); + dataEvent + .getDataItems() + .forEach( + dataItem -> { + DictWord dictWord = new DictWord(); + dictWord.setWord(dataItem.getName()); + String sign = DictWordType.NATURE_SPILT; + String suffixNature = DictWordType.getSuffixNature(dataItem.getType()); + String nature = + sign + dataItem.getModelId() + dataItem.getId() + suffixNature; + String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; + dictWord.setNature(nature); + dictWord.setNatureWithFrequency(natureWithFrequency); + if (EventType.ADD.equals(dataEvent.getEventType())) { + HanlpHelper.addToCustomDictionary(dictWord); + } else if (EventType.DELETE.equals(dataEvent.getEventType())) { + HanlpHelper.removeFromCustomDictionary(dictWord); + } else if (EventType.UPDATE.equals(dataEvent.getEventType())) { + HanlpHelper.removeFromCustomDictionary(dictWord); + dictWord.setWord(dataItem.getNewName()); + HanlpHelper.addToCustomDictionary(dictWord); + } + }); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java index de91070a6..630ab6a96 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.manager; - import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; @@ -13,9 +12,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -/** - * manager to handle the dimension - */ +/** manager to handle the dimension */ @Slf4j @Service public class DimensionYamlManager { @@ -25,8 +22,14 @@ public class DimensionYamlManager { return new ArrayList<>(); } return dimensions.stream() - .filter(dimension -> !dimension.getType().name().equalsIgnoreCase(IdentifyType.primary.name())) - .map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList()); + .filter( + dimension -> + !dimension + .getType() + .name() + .equalsIgnoreCase(IdentifyType.primary.name())) + .map(DimensionYamlManager::convert2DimensionYamlTpl) + .collect(Collectors.toList()); } public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) { @@ -36,5 +39,4 @@ public class DimensionYamlManager { dimensionYamlTpl.setOwners(dimension.getCreatedBy()); return dimensionYamlTpl; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java index d7bf12660..750d9d8a1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.headless.server.manager; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.FieldParam; import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import com.tencent.supersonic.headless.api.pojo.MetricParam; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; @@ -22,9 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -/** - * manager to handle the metric - */ +/** manager to handle the metric */ @Slf4j @Service public class MetricYamlManager { @@ -46,7 +44,8 @@ public class MetricYamlManager { metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy())); MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl(); if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) { - MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams(); + MetricDefineByMeasureParams metricDefineParams = + metric.getMetricDefineByMeasureParams(); metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr()); List measures = metricDefineParams.getMeasures(); metricTypeParamsYamlTpl.setMeasures( @@ -58,7 +57,8 @@ public class MetricYamlManager { metricTypeParamsYamlTpl.setFields( fields.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); } else if (MetricDefineType.METRIC.equals(metric.getMetricDefineType())) { - MetricDefineByMetricParams metricDefineByMetricParams = metric.getMetricDefineByMetricParams(); + MetricDefineByMetricParams metricDefineByMetricParams = + metric.getMetricDefineByMetricParams(); metricTypeParamsYamlTpl.setExpr(metricDefineByMetricParams.getExpr()); List metrics = metricDefineByMetricParams.getMetrics(); metricTypeParamsYamlTpl.setMetrics( @@ -88,5 +88,4 @@ public class MetricYamlManager { metricParamYamlTpl.setId(metricParam.getId()); return metricParamYamlTpl; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 975fae4ee..6ab181f5e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -23,14 +23,13 @@ import org.springframework.util.CollectionUtils; import java.util.Objects; import java.util.stream.Collectors; -/** - * manager to handle the model - */ +/** manager to handle the model */ @Service @Slf4j public class ModelYamlManager { - public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) { + public static synchronized DataModelYamlTpl convert2YamlObj( + ModelResp modelResp, DatabaseResp databaseResp) { ModelDetail modelDetail = modelResp.getModelDetail(); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor); @@ -38,12 +37,18 @@ public class ModelYamlManager { DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl(); dataModelYamlTpl.setType(databaseResp.getType()); BeanUtils.copyProperties(modelDetail, dataModelYamlTpl); - dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream().map(ModelYamlManager::convert) - .collect(Collectors.toList())); - dataModelYamlTpl.setDimensions(modelDetail.getDimensions().stream().map(ModelYamlManager::convert) - .collect(Collectors.toList())); - dataModelYamlTpl.setMeasures(modelDetail.getMeasures().stream().map(ModelYamlManager::convert) - .collect(Collectors.toList())); + dataModelYamlTpl.setIdentifiers( + modelDetail.getIdentifiers().stream() + .map(ModelYamlManager::convert) + .collect(Collectors.toList())); + dataModelYamlTpl.setDimensions( + modelDetail.getDimensions().stream() + .map(ModelYamlManager::convert) + .collect(Collectors.toList())); + dataModelYamlTpl.setMeasures( + modelDetail.getMeasures().stream() + .map(ModelYamlManager::convert) + .collect(Collectors.toList())); dataModelYamlTpl.setName(modelResp.getBizName()); dataModelYamlTpl.setSourceId(modelResp.getDatabaseId()); if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) { @@ -64,7 +69,8 @@ public class ModelYamlManager { dimensionYamlTpl.setExpr(dim.getBizName()); } if (dim.getTypeParams() != null) { - DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl = new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl = + new DimensionTimeTypeParamsTpl(); dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary()); dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity()); dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl); @@ -97,5 +103,4 @@ public class ModelYamlManager { measure.setIsCreateMetric(1); datasourceDetail.getMeasures().add(measure); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index b42296567..d909da451 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -29,8 +29,8 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Triple; import org.springframework.stereotype.Service; @@ -47,7 +47,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - @Slf4j @Service public class SemanticSchemaManager { @@ -65,16 +64,25 @@ public class SemanticSchemaManager { List dataModelYamlTpls = new ArrayList<>(); List metricYamlTpls = new ArrayList<>(); Map modelIdName = new HashMap<>(); - schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, - dataModelYamlTpls, metricYamlTpls, modelIdName); + schemaService.getSchemaYamlTpl( + semanticSchemaResp, + dimensionYamlTpls, + dataModelYamlTpls, + metricYamlTpls, + modelIdName); DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp(); semanticModel.setDatabase(DatabaseConverter.convert(databaseResp)); if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) { - semanticModel.setJoinRelations(getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); + semanticModel.setJoinRelations( + getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { - Map dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource) - .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); + Map dataSourceMap = + dataModelYamlTpls.stream() + .map(SemanticSchemaManager::getDatasource) + .collect( + Collectors.toMap( + DataSource::getName, item -> item, (k1, k2) -> k1)); semanticModel.setDatasourceMap(dataSourceMap); } if (!dimensionYamlTpls.isEmpty()) { @@ -90,12 +98,13 @@ public class SemanticSchemaManager { return semanticModel; } - public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception { + public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) + throws Exception { if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) { throw new Exception("semanticSchemaResp tag is empty"); } SemanticModel semanticModel = getSemanticModel(semanticSchemaResp); - //Map> dimensions = new HashMap<>(); + // Map> dimensions = new HashMap<>(); Map> tagMap = new HashMap<>(); for (TagResp tagResp : semanticSchemaResp.getTags()) { if (!tagMap.containsKey(tagResp.getModelId())) { @@ -103,8 +112,10 @@ public class SemanticSchemaManager { } tagMap.get(tagResp.getModelId()).add(tagResp); } - if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) { - for (Map.Entry entry : semanticModel.getDatasourceMap().entrySet()) { + if (Objects.nonNull(semanticModel.getDatasourceMap()) + && !semanticModel.getDatasourceMap().isEmpty()) { + for (Map.Entry entry : + semanticModel.getDatasourceMap().entrySet()) { List modelDimensions = new ArrayList<>(); if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) { semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions); @@ -122,16 +133,18 @@ public class SemanticSchemaManager { return semanticModel; } - private void addTagModel(TagResp tagResp, List modelDimensions, List modelMetrics) + private void addTagModel( + TagResp tagResp, List modelDimensions, List modelMetrics) throws Exception { TagDefineType tagDefineType = TagDefineType.valueOf(tagResp.getTagDefineType()); switch (tagDefineType) { case FIELD: case DIMENSION: if (TagDefineType.DIMENSION.equals(tagResp.getTagDefineType())) { - Optional modelDimension = modelDimensions.stream() - // .filter(d -> d.getBizName().equals(tagResp.getExpr())) - .findFirst(); + Optional modelDimension = + modelDimensions.stream() + // .filter(d -> d.getBizName().equals(tagResp.getExpr())) + .findFirst(); if (modelDimension.isPresent()) { modelDimension.get().setName(tagResp.getBizName()); return; @@ -152,13 +165,15 @@ public class SemanticSchemaManager { modelDimensions.add(dimension); return; case METRIC: - Optional modelMetric = modelMetrics.stream() - // .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr())) - .findFirst(); + Optional modelMetric = + modelMetrics.stream() + // .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr())) + .findFirst(); if (modelMetric.isPresent()) { modelMetric.get().setName(tagResp.getBizName()); } else { - throw new Exception(String.format("tag [{}] cant find the metric", tagResp.getBizName())); + throw new Exception( + String.format("tag [{}] cant find the metric", tagResp.getBizName())); } return; default: @@ -174,21 +189,37 @@ public class SemanticSchemaManager { } public static DataSource getDatasource(final DataModelYamlTpl d) { - DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId()) - .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery()) - .identifiers(getIdentify(d.getIdentifiers())).measures(getMeasureParams(d.getMeasures())) - .dimensions(getDimensions(d.getDimensions())).build(); + DataSource datasource = + DataSource.builder() + .id(d.getId()) + .sourceId(d.getSourceId()) + .type(d.getType()) + .sqlQuery(d.getSqlQuery()) + .name(d.getName()) + .tableQuery(d.getTableQuery()) + .identifiers(getIdentify(d.getIdentifiers())) + .measures(getMeasureParams(d.getMeasures())) + .dimensions(getDimensions(d.getDimensions())) + .build(); datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions())); if (Objects.nonNull(d.getModelSourceTypeEnum())) { datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); } if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) { - Set measures = datasource.getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); + Set measures = + datasource.getMeasures().stream() + .map(mm -> mm.getName()) + .collect(Collectors.toSet()); for (Field f : d.getFields()) { if (!measures.contains(f.getFieldName())) { - datasource.getMeasures() - .add(Measure.builder().expr(f.getFieldName()).name(f.getFieldName()).agg("").build()); + datasource + .getMeasures() + .add( + Measure.builder() + .expr(f.getFieldName()) + .name(f.getFieldName()) + .agg("") + .build()); } } } @@ -196,9 +227,12 @@ public class SemanticSchemaManager { } private static String getDataSourceAggTime(List dimensions) { - Optional timeDimension = dimensions.stream() - .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst(); - if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) { + Optional timeDimension = + dimensions.stream() + .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) + .findFirst(); + if (timeDimension.isPresent() + && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) { return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity(); } return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE; @@ -217,7 +251,8 @@ public class SemanticSchemaManager { return metrics; } - private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) { + private static MetricTypeParams getMetricTypeParams( + MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) { MetricTypeParams metricTypeParams = new MetricTypeParams(); metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr()); metricTypeParams.setFieldMetric(false); @@ -294,7 +329,8 @@ public class SemanticSchemaManager { if (Objects.nonNull(dimensionYamlTpl.getExt())) { dimension.setExt(dimensionYamlTpl.getExt()); } - dimension.setDimensionTimeTypeParams(getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams())); + dimension.setDimensionTimeTypeParams( + getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams())); dimensions.add(dimension); } return dimensions; @@ -304,7 +340,8 @@ public class SemanticSchemaManager { DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) { DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams(); if (dimensionTimeTypeParamsTpl != null) { - dimensionTimeTypeParams.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity()); + dimensionTimeTypeParams.setTimeGranularity( + dimensionTimeTypeParamsTpl.getTimeGranularity()); dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary()); } return dimensionTimeTypeParams; @@ -321,24 +358,38 @@ public class SemanticSchemaManager { return identifies; } - private static List getJoinRelation(List modelRelas, Map modelIdName) { + private static List getJoinRelation( + List modelRelas, Map modelIdName) { List joinRelations = new ArrayList<>(); - modelRelas.stream().forEach(r -> { - if (modelIdName.containsKey(r.getFromModelId()) && modelIdName.containsKey(r.getToModelId())) { - JoinRelation joinRelation = JoinRelation.builder().left(modelIdName.get(r.getFromModelId())) - .right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build(); - List> conditions = new ArrayList<>(); - r.getJoinConditions().stream().forEach(rr -> { - if (FilterOperatorEnum.isValueCompare(rr.getOperator())) { - conditions.add( - Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField())); - } - }); - joinRelation.setId(r.getId()); - joinRelation.setJoinCondition(conditions); - joinRelations.add(joinRelation); - } - }); + modelRelas.stream() + .forEach( + r -> { + if (modelIdName.containsKey(r.getFromModelId()) + && modelIdName.containsKey(r.getToModelId())) { + JoinRelation joinRelation = + JoinRelation.builder() + .left(modelIdName.get(r.getFromModelId())) + .right(modelIdName.get(r.getToModelId())) + .joinType(r.getJoinType()) + .build(); + List> conditions = new ArrayList<>(); + r.getJoinConditions().stream() + .forEach( + rr -> { + if (FilterOperatorEnum.isValueCompare( + rr.getOperator())) { + conditions.add( + Triple.of( + rr.getLeftField(), + rr.getOperator().getValue(), + rr.getRightField())); + } + }); + joinRelation.setId(r.getId()); + joinRelation.setJoinCondition(conditions); + joinRelations.add(joinRelation); + } + }); return joinRelations; } @@ -348,11 +399,14 @@ public class SemanticSchemaManager { } } - public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception { + public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) + throws Exception { if (schema != null) { String dataSourceName = datasourceYamlTpl.getName(); - Optional> datasourceYamlTplMap = schema.getDatasource().entrySet().stream() - .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst(); + Optional> datasourceYamlTplMap = + schema.getDatasource().entrySet().stream() + .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)) + .findFirst(); if (datasourceYamlTplMap.isPresent()) { datasourceYamlTplMap.get().setValue(datasourceYamlTpl); } else { @@ -361,11 +415,14 @@ public class SemanticSchemaManager { } } - public static void update(SemanticSchema schema, String datasourceBizName, List dimensionYamlTpls) + public static void update( + SemanticSchema schema, String datasourceBizName, List dimensionYamlTpls) throws Exception { if (schema != null) { - Optional>> datasourceYamlTplMap = schema.getDimension().entrySet() - .stream().filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst(); + Optional>> datasourceYamlTplMap = + schema.getDimension().entrySet().stream() + .filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)) + .findFirst(); if (datasourceYamlTplMap.isPresent()) { updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue()); } else { @@ -376,11 +433,13 @@ public class SemanticSchemaManager { } } - private static void updateDimension(List dimensionYamlTpls, List dimensions) { + private static void updateDimension( + List dimensionYamlTpls, List dimensions) { if (CollectionUtils.isEmpty(dimensionYamlTpls)) { return; } - Set toAdd = dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet()); + Set toAdd = + dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet()); Iterator iterator = dimensions.iterator(); while (iterator.hasNext()) { Dimension cur = iterator.next(); @@ -395,7 +454,8 @@ public class SemanticSchemaManager { if (CollectionUtils.isEmpty(metricYamlTpls)) { return; } - Set toAdd = metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet()); + Set toAdd = + metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet()); Iterator iterator = metrics.iterator(); while (iterator.hasNext()) { Metric cur = iterator.next(); @@ -405,5 +465,4 @@ public class SemanticSchemaManager { } metrics.addAll(metricYamlTpls); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/AppDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/AppDO.java index fbca95e9b..d40e8d4d8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/AppDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/AppDO.java @@ -37,5 +37,4 @@ public class AppDO { private Date createdAt; private Date updatedAt; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java index 9e7f191f2..2fb0bb5e1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java @@ -10,38 +10,28 @@ import java.util.Date; @Data @TableName("s2_canvas") public class CanvasDO { - /** - */ + /** */ @TableId(type = IdType.AUTO) private Long id; - /** - */ + /** */ private Long domainId; - /** - * datasource、dimension、metric - */ + /** datasource、dimension、metric */ private String type; - /** - */ + /** */ private Date createdAt; - /** - */ + /** */ private String createdBy; - /** - */ + /** */ private Date updatedAt; - /** - */ + /** */ private String updatedBy; - /** - * config detail - */ + /** config detail */ private String config; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ClassDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ClassDO.java index ea5ac1c60..a44f739b4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ClassDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ClassDO.java @@ -22,21 +22,16 @@ public class ClassDO { private String description; private Long parentId; - /** - * 分类状态 - */ + /** 分类状态 */ private Integer status; - /** - * METRIC、DIMENSION、TAG - */ + /** METRIC、DIMENSION、TAG */ private String type; + private String itemIds; private Date createdAt; private String createdBy; private Date updatedAt; private String updatedBy; - - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CollectDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CollectDO.java index ce748a38f..d1b62dd01 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CollectDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CollectDO.java @@ -12,9 +12,7 @@ import java.io.Serializable; import java.time.LocalDateTime; /** - *

* 收藏项表 - *

* * @author yannsu * @since 2023-11-09 03:49:33 @@ -26,39 +24,27 @@ public class CollectDO implements Serializable { private static final long serialVersionUID = 1L; - /** - * 收藏项ID - */ + /** 收藏项ID */ @TableId(value = "id", type = IdType.AUTO) private Long id; - /** - * 收藏的类型 metric,dimension,tag - */ + /** 收藏的类型 metric,dimension,tag */ @TableField("type") private String type; - /** - * 收藏人 - */ + /** 收藏人 */ @TableField("username") private String username; - /** - * 收藏ID - */ + /** 收藏ID */ @TableField("collect_id") private Long collectId; - /** - * 创建时间 - */ + /** 创建时间 */ @TableField(value = "create_time", fill = FieldFill.INSERT) private LocalDateTime createTime; - /** - * 更新时间 - */ + /** 更新时间 */ @TableField(value = "update_time", fill = FieldFill.INSERT_UPDATE) private LocalDateTime updateTime; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java index e6acdfefc..2fbbaff64 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java @@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.dataobject; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import java.util.Date; import lombok.Data; +import java.util.Date; + @Data @TableName("s2_data_set") public class DataSetDO { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDO.java index 6be1920fb..a48955f90 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDO.java @@ -10,85 +10,56 @@ import java.util.Date; @Data @TableName("s2_database") public class DatabaseDO { - /** - * - */ + /** */ @TableId(type = IdType.AUTO) private Long id; - /** - * 名称 - */ + /** 名称 */ private String name; - /** - * 描述 - */ + /** 描述 */ private String description; - /** - * - */ + /** */ private String version; - /** - * 类型 mysql,clickhouse,tdw - */ + /** 类型 mysql,clickhouse,tdw */ private String type; - /** - * 创建时间 - */ + /** 创建时间 */ private Date createdAt; - /** - * 创建人 - */ + /** 创建人 */ private String createdBy; - /** - * 更新时间 - */ + /** 更新时间 */ private Date updatedAt; - /** - * 更新人 - */ + /** 更新人 */ private String updatedBy; - /** - * - */ + /** */ private String admin; - /** - * - */ + /** */ private String viewer; - /** - * 配置信息 - */ + /** 配置信息 */ private String config; - /** - * - * @return id - */ + /** @return id */ public Long getId() { return id; } - /** - * - * @param id - */ + /** @param id */ public void setId(Long id) { this.id = id; } /** * 名称 + * * @return name 名称 */ public String getName() { @@ -97,6 +68,7 @@ public class DatabaseDO { /** * 名称 + * * @param name 名称 */ public void setName(String name) { @@ -105,6 +77,7 @@ public class DatabaseDO { /** * 描述 + * * @return description 描述 */ public String getDescription() { @@ -113,30 +86,26 @@ public class DatabaseDO { /** * 描述 + * * @param description 描述 */ public void setDescription(String description) { this.description = description == null ? null : description.trim(); } - /** - * - * @return version - */ + /** @return version */ public String getVersion() { return version; } - /** - * - * @param version - */ + /** @param version */ public void setVersion(String version) { this.version = version == null ? null : version.trim(); } /** * 类型 mysql,clickhouse,tdw + * * @return type 类型 mysql,clickhouse,tdw */ public String getType() { @@ -145,6 +114,7 @@ public class DatabaseDO { /** * 类型 mysql,clickhouse,tdw + * * @param type 类型 mysql,clickhouse,tdw */ public void setType(String type) { @@ -153,6 +123,7 @@ public class DatabaseDO { /** * 创建时间 + * * @return created_at 创建时间 */ public Date getCreatedAt() { @@ -161,6 +132,7 @@ public class DatabaseDO { /** * 创建时间 + * * @param createdAt 创建时间 */ public void setCreatedAt(Date createdAt) { @@ -169,6 +141,7 @@ public class DatabaseDO { /** * 创建人 + * * @return created_by 创建人 */ public String getCreatedBy() { @@ -177,6 +150,7 @@ public class DatabaseDO { /** * 创建人 + * * @param createdBy 创建人 */ public void setCreatedBy(String createdBy) { @@ -185,6 +159,7 @@ public class DatabaseDO { /** * 更新时间 + * * @return updated_at 更新时间 */ public Date getUpdatedAt() { @@ -193,6 +168,7 @@ public class DatabaseDO { /** * 更新时间 + * * @param updatedAt 更新时间 */ public void setUpdatedAt(Date updatedAt) { @@ -201,6 +177,7 @@ public class DatabaseDO { /** * 更新人 + * * @return updated_by 更新人 */ public String getUpdatedBy() { @@ -209,46 +186,36 @@ public class DatabaseDO { /** * 更新人 + * * @param updatedBy 更新人 */ public void setUpdatedBy(String updatedBy) { this.updatedBy = updatedBy == null ? null : updatedBy.trim(); } - /** - * - * @return admin - */ + /** @return admin */ public String getAdmin() { return admin; } - /** - * - * @param admin - */ + /** @param admin */ public void setAdmin(String admin) { this.admin = admin == null ? null : admin.trim(); } - /** - * - * @return viewer - */ + /** @return viewer */ public String getViewer() { return viewer; } - /** - * - * @param viewer - */ + /** @param viewer */ public void setViewer(String viewer) { this.viewer = viewer == null ? null : viewer.trim(); } /** * 配置信息 + * * @return config 配置信息 */ public String getConfig() { @@ -257,9 +224,10 @@ public class DatabaseDO { /** * 配置信息 + * * @param config 配置信息 */ public void setConfig(String config) { this.config = config == null ? null : config.trim(); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDOExample.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDOExample.java index 6d56191e7..d3ce7cacb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDOExample.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDOExample.java @@ -5,101 +5,64 @@ import java.util.Date; import java.util.List; public class DatabaseDOExample { - /** - * s2_database - */ + /** s2_database */ protected String orderByClause; - /** - * s2_database - */ + /** s2_database */ protected boolean distinct; - /** - * s2_database - */ + /** s2_database */ protected List oredCriteria; - /** - * s2_database - */ + /** s2_database */ protected Integer limitStart; - /** - * s2_database - */ + /** s2_database */ protected Integer limitEnd; - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public DatabaseDOExample() { oredCriteria = new ArrayList(); } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setOrderByClause(String orderByClause) { this.orderByClause = orderByClause; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public String getOrderByClause() { return orderByClause; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setDistinct(boolean distinct) { this.distinct = distinct; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public boolean isDistinct() { return distinct; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public List getOredCriteria() { return oredCriteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void or(Criteria criteria) { oredCriteria.add(criteria); } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Criteria or() { Criteria criteria = createCriteriaInternal(); oredCriteria.add(criteria); return criteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Criteria createCriteria() { Criteria criteria = createCriteriaInternal(); if (oredCriteria.size() == 0) { @@ -108,60 +71,40 @@ public class DatabaseDOExample { return criteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ protected Criteria createCriteriaInternal() { Criteria criteria = new Criteria(); return criteria; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void clear() { oredCriteria.clear(); orderByClause = null; distinct = false; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setLimitStart(Integer limitStart) { - this.limitStart=limitStart; + this.limitStart = limitStart; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Integer getLimitStart() { return limitStart; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public void setLimitEnd(Integer limitEnd) { - this.limitEnd=limitEnd; + this.limitEnd = limitEnd; } - /** - * - * @mbg.generated - */ + /** @mbg.generated */ public Integer getLimitEnd() { return limitEnd; } - /** - * s2_database null - */ + /** s2_database null */ protected abstract static class GeneratedCriteria { protected List criteria; @@ -196,7 +139,8 @@ public class DatabaseDOExample { criteria.add(new Criterion(condition, value)); } - protected void addCriterion(String condition, Object value1, Object value2, String property) { + protected void addCriterion( + String condition, Object value1, Object value2, String property) { if (value1 == null || value2 == null) { throw new RuntimeException("Between values for " + property + " cannot be null"); } @@ -944,9 +888,7 @@ public class DatabaseDOExample { } } - /** - * s2_database - */ + /** s2_database */ public static class Criteria extends GeneratedCriteria { protected Criteria() { @@ -954,9 +896,7 @@ public class DatabaseDOExample { } } - /** - * s2_database null - */ + /** s2_database null */ public static class Criterion { private String condition; @@ -1029,7 +969,8 @@ public class DatabaseDOExample { this(condition, value, null); } - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { + protected Criterion( + String condition, Object value, Object secondValue, String typeHandler) { super(); this.condition = condition; this.value = value; @@ -1042,4 +983,4 @@ public class DatabaseDOExample { this(condition, value, secondValue, null); } } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DateInfoDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DateInfoDO.java index da8493fd4..063a2e13f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DateInfoDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DateInfoDO.java @@ -15,6 +15,4 @@ public class DateInfoDO { private String createdBy; private String updatedBy; private String datePeriod; - - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java index 61316dc79..f08a2817f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java @@ -12,6 +12,7 @@ import java.util.Date; public class DictConfDO { @TableId(type = IdType.AUTO) private Long id; + private String description; private String type; private Long itemId; @@ -19,4 +20,4 @@ public class DictConfDO { private String status; private Date createdAt; private String createdBy; -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java index 25ee9db92..92484f532 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java @@ -12,6 +12,7 @@ import java.util.Date; public class DictTaskDO { @TableId(type = IdType.AUTO) private Long id; + private String name; private String description; private String type; @@ -21,5 +22,4 @@ public class DictTaskDO { private Date createdAt; private String createdBy; private Long elapsedMs; - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDO.java index 0597c812f..664a62f55 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDO.java @@ -10,75 +10,46 @@ import java.util.Date; @Data @TableName("s2_domain") public class DomainDO { - /** - * 自增ID - */ + /** 自增ID */ @TableId(type = IdType.AUTO) private Long id; - /** - * 主题域名称 - */ + /** 主题域名称 */ private String name; - /** - * 内部名称 - */ + /** 内部名称 */ private String bizName; - /** - * 父主题域ID - */ + /** 父主题域ID */ private Long parentId; - /** - * 主题域状态 - */ + /** 主题域状态 */ private Integer status; - /** - * 创建时间 - */ + /** 创建时间 */ private Date createdAt; - /** - * 创建人 - */ + /** 创建人 */ private String createdBy; - /** - * 更新时间 - */ + /** 更新时间 */ private Date updatedAt; - /** - * 更新人 - */ + /** 更新人 */ private String updatedBy; - /** - * 主题域管理员 - */ + /** 主题域管理员 */ private String admin; - /** - * 主题域管理员组织 - */ + /** 主题域管理员组织 */ private String adminOrg; - /** - * 主题域是否公开 - */ + /** 主题域是否公开 */ private Integer isOpen; - /** - * 主题域可用用户 - */ + /** 主题域可用用户 */ private String viewer; - /** - * 主题域可用组织 - */ + /** 主题域可用组织 */ private String viewOrg; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDOExample.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDOExample.java index 4c27334bc..bee796f64 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDOExample.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDOExample.java @@ -5,92 +5,64 @@ import java.util.Date; import java.util.List; public class DomainDOExample { - /** - * s2_domain - */ + /** s2_domain */ protected String orderByClause; - /** - * s2_domain - */ + /** s2_domain */ protected boolean distinct; - /** - * s2_domain - */ + /** s2_domain */ protected List oredCriteria; - /** - * s2_domain - */ + /** s2_domain */ protected Integer limitStart; - /** - * s2_domain - */ + /** s2_domain */ protected Integer limitEnd; - /** - * @mbg.generated - */ + /** @mbg.generated */ public DomainDOExample() { oredCriteria = new ArrayList(); } - /** - * @mbg.generated - */ + /** @mbg.generated */ public void setOrderByClause(String orderByClause) { this.orderByClause = orderByClause; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public String getOrderByClause() { return orderByClause; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public void setDistinct(boolean distinct) { this.distinct = distinct; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public boolean isDistinct() { return distinct; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public List getOredCriteria() { return oredCriteria; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public void or(Criteria criteria) { oredCriteria.add(criteria); } - /** - * @mbg.generated - */ + /** @mbg.generated */ public Criteria or() { Criteria criteria = createCriteriaInternal(); oredCriteria.add(criteria); return criteria; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public Criteria createCriteria() { Criteria criteria = createCriteriaInternal(); if (oredCriteria.size() == 0) { @@ -99,54 +71,40 @@ public class DomainDOExample { return criteria; } - /** - * @mbg.generated - */ + /** @mbg.generated */ protected Criteria createCriteriaInternal() { Criteria criteria = new Criteria(); return criteria; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public void clear() { oredCriteria.clear(); orderByClause = null; distinct = false; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public void setLimitStart(Integer limitStart) { this.limitStart = limitStart; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public Integer getLimitStart() { return limitStart; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public void setLimitEnd(Integer limitEnd) { this.limitEnd = limitEnd; } - /** - * @mbg.generated - */ + /** @mbg.generated */ public Integer getLimitEnd() { return limitEnd; } - /** - * s2_domain null - */ + /** s2_domain null */ protected abstract static class GeneratedCriteria { protected List criteria; @@ -181,7 +139,8 @@ public class DomainDOExample { criteria.add(new Criterion(condition, value)); } - protected void addCriterion(String condition, Object value1, Object value2, String property) { + protected void addCriterion( + String condition, Object value1, Object value2, String property) { if (value1 == null || value2 == null) { throw new RuntimeException("Between values for " + property + " cannot be null"); } @@ -1109,9 +1068,7 @@ public class DomainDOExample { } } - /** - * s2_domain - */ + /** s2_domain */ public static class Criteria extends GeneratedCriteria { protected Criteria() { @@ -1119,9 +1076,7 @@ public class DomainDOExample { } } - /** - * s2_domain null - */ + /** s2_domain null */ public static class Criterion { private String condition; @@ -1162,7 +1117,8 @@ public class DomainDOExample { this(condition, value, null); } - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { + protected Criterion( + String condition, Object value, Object secondValue, String typeHandler) { super(); this.condition = condition; this.value = value; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricDO.java index 6f6759039..74d3df4b4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricDO.java @@ -15,89 +15,55 @@ public class MetricDO { @TableId(type = IdType.AUTO) private Long id; - /** - * 主体域ID - */ + /** 主体域ID */ private Long modelId; - /** - * 指标名称 - */ + /** 指标名称 */ private String name; - /** - * 字段名称 - */ + /** 字段名称 */ private String bizName; - /** - * 描述 - */ + /** 描述 */ private String description; - /** - * 指标状态,0正常,1下架,2删除 - */ + /** 指标状态,0正常,1下架,2删除 */ private Integer status; - /** - * 敏感级别 - */ + /** 敏感级别 */ private Integer sensitiveLevel; - /** - * 指标类型 proxy,expr - */ + /** 指标类型 proxy,expr */ private String type; - /** - * 创建时间 - */ + /** 创建时间 */ private Date createdAt; - /** - * 创建人 - */ + /** 创建人 */ private String createdBy; - /** - * 更新时间 - */ + /** 更新时间 */ private Date updatedAt; - /** - * 更新人 - */ + /** 更新人 */ private String updatedBy; - /** - * 数值类型 - */ + /** 数值类型 */ private String dataFormatType; - /** - * 数值类型参数 - */ + /** 数值类型参数 */ private String dataFormat; - /** - * - */ + /** */ private String alias; - /** - * - */ + /** */ private String classifications; - /** - * - */ + /** */ private String relateDimensions; - /** - * 类型参数 - */ + /** 类型参数 */ private String typeParams; private String ext; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricQueryDefaultConfigDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricQueryDefaultConfigDO.java index 99824ba73..06d33b622 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricQueryDefaultConfigDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricQueryDefaultConfigDO.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.dataobject; - import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -12,7 +11,6 @@ import java.util.Date; @TableName("s2_metric_query_default_config") public class MetricQueryDefaultConfigDO { - @TableId(value = "id", type = IdType.AUTO) private Long id; @@ -35,5 +33,4 @@ public class MetricQueryDefaultConfigDO { private Date updatedAt; private String updatedBy; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java index eb5eeca2a..096fccf69 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java @@ -59,5 +59,4 @@ public class ModelDO { private String sourceType; private String ext; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelRelaDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelRelaDO.java index a48fcf425..7827945dc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelRelaDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelRelaDO.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.dataobject; - import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -22,5 +21,4 @@ public class ModelRelaDO { private String joinType; private String joinCondition; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryRuleDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryRuleDO.java index e3e049c7e..5c34b3adf 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryRuleDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryRuleDO.java @@ -14,75 +14,45 @@ public class QueryRuleDO { @TableId(type = IdType.AUTO) private Long id; - /** - * dataSetID - */ + /** dataSetID */ private Long dataSetId; - /** - * 规则的优先级, 0-系统默认规则 - */ + /** 规则的优先级, 0-系统默认规则 */ private Integer priority; - /** - * 规则类型 - */ + /** 规则类型 */ private String ruleType; - /** - * 规则名称 - */ + /** 规则名称 */ private String name; - /** - * 规则业务名称 - */ + /** 规则业务名称 */ private String bizName; - /** - * 描述 - */ + /** 描述 */ private String description; - /** - * 具体规则信息 - */ + /** 具体规则信息 */ private String rule; - /** - * 规则输出信息 - */ + /** 规则输出信息 */ private String action; - /** - * 状态,0-正常,1-下线,2-删除 - */ + /** 状态,0-正常,1-下线,2-删除 */ private Integer status; - /** - * 创建时间 - */ + /** 创建时间 */ private Date createdAt; - /** - * 创建人 - */ + /** 创建人 */ private String createdBy; - /** - * 更新时间 - */ + /** 更新时间 */ private Date updatedAt; - /** - * 更新人 - */ + /** 更新人 */ private String updatedBy; - /** - * 扩展信息 - */ + /** 扩展信息 */ private String ext; - - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java index 66f8604c8..339048948 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java @@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.dataobject; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import java.util.Date; import lombok.Data; +import java.util.Date; + @Data @TableName("s2_tag") public class TagDO { @@ -13,34 +14,21 @@ public class TagDO { @TableId(type = IdType.AUTO) private Long id; - /** - * itemID - */ + /** itemID */ private Long itemId; - /** - * 标签类型 - */ + /** 标签类型 */ private String type; - /** - * 创建时间 - */ + /** 创建时间 */ private Date createdAt; - /** - * 创建人 - */ + /** 创建人 */ private String createdBy; - /** - * 更新时间 - */ + /** 更新时间 */ private Date updatedAt; - /** - * 更新人 - */ + /** 更新人 */ private String updatedBy; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagObjectDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagObjectDO.java index e5ce6a00c..ee8d6cd56 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagObjectDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagObjectDO.java @@ -14,58 +14,36 @@ public class TagObjectDO { @TableId(type = IdType.AUTO) private Long id; - /** - * 关联到某个主题域下 - */ + /** 关联到某个主题域下 */ private Long domainId; - /** - * 标签对象名称 - */ + /** 标签对象名称 */ private String name; - /** - * 标签对象业务名称 - */ + /** 标签对象业务名称 */ private String bizName; - /** - * 描述 - */ + /** 描述 */ private String description; - /** - * 状态,0正常,1下架,2删除 - */ + /** 状态,0正常,1下架,2删除 */ private Integer status; - /** - * 敏感级别 - */ + /** 敏感级别 */ private Integer sensitiveLevel; - /** - * 创建时间 - */ + /** 创建时间 */ private Date createdAt; - /** - * 创建人 - */ + /** 创建人 */ private String createdBy; - /** - * 更新时间 - */ + /** 更新时间 */ private Date updatedAt; - /** - * 更新人 - */ + /** 更新人 */ private String updatedBy; - /** - * 扩展信息 - */ + /** 扩展信息 */ private String ext; -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermDO.java index ab357b98f..79ae5dfea 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermDO.java @@ -33,5 +33,4 @@ public class TermDO { private Date updatedAt; private String updatedBy; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/AppMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/AppMapper.java index 5584c2d6e..67fb28b93 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/AppMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/AppMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.AppDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface AppMapper extends BaseMapper { - -} +public interface AppMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java index c453b38e4..f9589000d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface CanvasDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface CanvasDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ClassMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ClassMapper.java index fc1857326..760f4a896 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ClassMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ClassMapper.java @@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ClassDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ClassMapper extends BaseMapper { -} \ No newline at end of file +public interface ClassMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CollectMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CollectMapper.java index 23867b876..5753eb1e9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CollectMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CollectMapper.java @@ -1,19 +1,14 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; import org.apache.ibatis.annotations.Mapper; /** - *

* 收藏项表 Mapper 接口 - *

* * @author yannsu * @since 2023-11-09 03:49:33 */ @Mapper -public interface CollectMapper extends BaseMapper { - -} +public interface CollectMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java index 7259e1c18..220d1a1fe 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java @@ -5,7 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DataSetDOMapper extends BaseMapper { - - -} +public interface DataSetDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DatabaseDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DatabaseDOMapper.java index 925c75f20..21fb39401 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DatabaseDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DatabaseDOMapper.java @@ -1,11 +1,8 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DatabaseDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface DatabaseDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DateInfoMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DateInfoMapper.java index eba0c686f..2d0646ba1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DateInfoMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DateInfoMapper.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import org.apache.ibatis.annotations.Mapper; @@ -13,4 +12,4 @@ public interface DateInfoMapper { Boolean upsertDateInfo(DateInfoDO dateInfoDO); List getDateInfos(ItemDateFilter itemDateFilter); -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java index b3e9f890b..c9d7cee7a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java @@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DictConfMapper extends BaseMapper { -} +public interface DictConfMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java index a13c04055..cf4a54118 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java @@ -1,11 +1,8 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DictTaskMapper extends BaseMapper { - -} \ No newline at end of file +public interface DictTaskMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java index 03504b989..746afe6a5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java @@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; -import java.util.List; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + @Mapper public interface DimensionDOCustomMapper { @@ -18,5 +19,4 @@ public interface DimensionDOCustomMapper { List query(DimensionFilter dimensionFilter); List queryDimensions(DimensionsFilter dimensionsFilter); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOMapper.java index 5f8859742..bd5239aff 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DimensionDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface DimensionDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DomainDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DomainDOMapper.java index 24fdf1436..4012fd1e6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DomainDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DomainDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DomainDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface DomainDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java index cfcfd4f8f..651e927d3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import org.apache.ibatis.annotations.Mapper; + import java.util.List; @Mapper diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOMapper.java index fd3a07336..3318912ea 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface MetricDOMapper extends BaseMapper { - -} \ No newline at end of file +public interface MetricDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricQueryDefaultConfigDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricQueryDefaultConfigDOMapper.java index 8a48c0130..3f1db5bfa 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricQueryDefaultConfigDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricQueryDefaultConfigDOMapper.java @@ -1,14 +1,8 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface MetricQueryDefaultConfigDOMapper extends BaseMapper { - - - - -} \ No newline at end of file +public interface MetricQueryDefaultConfigDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOCustomMapper.java index 8d4a8fe08..5cafc8714 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOCustomMapper.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import org.apache.ibatis.annotations.Mapper; @@ -10,5 +9,4 @@ import java.util.List; public interface ModelDOCustomMapper { void batchUpdateStatus(List modelDOS); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOMapper.java index fdeb31682..9521bcf7b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOMapper.java @@ -5,7 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ModelDOMapper extends BaseMapper { - - -} \ No newline at end of file +public interface ModelDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelRelaDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelRelaDOMapper.java index f6ca06e13..ae062d209 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelRelaDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelRelaDOMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelRelaDO import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ModelRelaDOMapper extends BaseMapper { - -} +public interface ModelRelaDOMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/QueryRuleMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/QueryRuleMapper.java index e0db70589..6556c0682 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/QueryRuleMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/QueryRuleMapper.java @@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.QueryRuleDO import org.apache.ibatis.annotations.Mapper; @Mapper -public interface QueryRuleMapper extends BaseMapper { -} +public interface QueryRuleMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java index 967c0ff15..e9475617e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.server.persistence.mapper; - import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; -import java.util.List; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + @Mapper public interface StatMapper { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java index b9abd5117..b5fbe339e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java @@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; import com.tencent.supersonic.headless.server.pojo.TagFilter; -import java.util.List; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + @Mapper public interface TagCustomMapper { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java index 1222762e3..30388d123 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java @@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface TagMapper extends BaseMapper { - -} +public interface TagMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagObjectMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagObjectMapper.java index c34b7300f..72d64e394 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagObjectMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagObjectMapper.java @@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagObjectDO import org.apache.ibatis.annotations.Mapper; @Mapper -public interface TagObjectMapper extends BaseMapper { -} +public interface TagObjectMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermMapper.java index f757a95f8..dcfb85b1d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermMapper.java @@ -5,7 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TermDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface TermMapper extends BaseMapper { - - -} \ No newline at end of file +public interface TermMapper extends BaseMapper {} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DateInfoRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DateInfoRepository.java index 5bf403ea1..6d1ac9002 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DateInfoRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DateInfoRepository.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.repository; - import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.request.DateInfoReq; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; @@ -12,5 +11,4 @@ public interface DateInfoRepository { Integer upsertDateInfo(List dateInfoReqs); List getDateInfos(ItemDateFilter itemDateFilter); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java index 84ee389e5..82153d129 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java @@ -1,10 +1,9 @@ package com.tencent.supersonic.headless.server.persistence.repository; - import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; - import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; + import java.util.List; public interface DimensionRepository { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DomainRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DomainRepository.java index 230be40dc..4ee3f9db7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DomainRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DomainRepository.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.repository; - import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO; import java.util.List; @@ -16,5 +15,4 @@ public interface DomainRepository { List getDomainList(); DomainDO getDomainById(Long id); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java index d7fe31368..69a750282 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java @@ -1,11 +1,10 @@ package com.tencent.supersonic.headless.server.persistence.repository; - import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; import com.tencent.supersonic.headless.server.pojo.MetricFilter; - import com.tencent.supersonic.headless.server.pojo.MetricsFilter; + import java.util.List; public interface MetricRepository { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ModelRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ModelRepository.java index 70110b7fa..7cd2169d6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ModelRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ModelRepository.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.repository; - import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.pojo.ModelFilter; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/QueryRuleRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/QueryRuleRepository.java index 1f228eacd..c98f10e7a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/QueryRuleRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/QueryRuleRepository.java @@ -14,5 +14,4 @@ public interface QueryRuleRepository { QueryRuleDO getQueryRuleById(Long id); List getQueryRules(QueryRuleFilter filter); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java index 8f9fac9d6..fd890b89f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.persistence.repository; import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; + import java.util.List; public interface StatRepository { @@ -12,4 +13,4 @@ public interface StatRepository { List getStatInfo(ItemUseReq itemUseCommend); List getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend); -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java index c306e0c91..304df9d73 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.server.persistence.repository; - import com.tencent.supersonic.headless.api.pojo.request.TagDeleteReq; import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; import com.tencent.supersonic.headless.server.pojo.TagFilter; -import java.util.List; +import java.util.List; public interface TagRepository { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ClassRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ClassRepositoryImpl.java index afb9dd8c7..2488471eb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ClassRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ClassRepositoryImpl.java @@ -75,4 +75,4 @@ public class ClassRepositoryImpl implements ClassRepository { QueryWrapper wrapper = new QueryWrapper(); return mapper.selectList(wrapper); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DateInfoRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DateInfoRepositoryImpl.java index 85ae4b17a..bf3cee7b4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DateInfoRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DateInfoRepositoryImpl.java @@ -24,11 +24,9 @@ import java.util.concurrent.TimeUnit; @Repository public class DateInfoRepositoryImpl implements DateInfoRepository { - private ObjectMapper mapper = new ObjectMapper(); - @Autowired - private DateInfoMapper dateInfoMapper; + @Autowired private DateInfoMapper dateInfoMapper; @Override public Integer upsertDateInfo(List dateInfoCommends) { @@ -38,25 +36,30 @@ public class DateInfoRepositoryImpl implements DateInfoRepository { return 0; } - dateInfoCommends.stream().forEach(commend -> { - DateInfoDO dateInfoDO = new DateInfoDO(); - BeanUtils.copyProperties(commend, dateInfoDO); - try { - dateInfoDO.setUnavailableDateList(mapper.writeValueAsString(commend.getUnavailableDateList())); - dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER); - dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER); - } catch (JsonProcessingException e) { - log.info("e,", e); - } - dateInfoDOList.add(dateInfoDO); - }); + dateInfoCommends.stream() + .forEach( + commend -> { + DateInfoDO dateInfoDO = new DateInfoDO(); + BeanUtils.copyProperties(commend, dateInfoDO); + try { + dateInfoDO.setUnavailableDateList( + mapper.writeValueAsString( + commend.getUnavailableDateList())); + dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER); + dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER); + } catch (JsonProcessingException e) { + log.info("e,", e); + } + dateInfoDOList.add(dateInfoDO); + }); return batchUpsert(dateInfoDOList); } @Override public List getDateInfos(ItemDateFilter itemDateFilter) { - if (Objects.nonNull(itemDateFilter) && CollectionUtils.isEmpty(itemDateFilter.getItemIds())) { + if (Objects.nonNull(itemDateFilter) + && CollectionUtils.isEmpty(itemDateFilter.getItemIds())) { return new ArrayList<>(); } return dateInfoMapper.getDateInfos(itemDateFilter); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java index 6a877165c..a5b3f2ab7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl; - import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; @@ -37,8 +36,11 @@ public class DictRepositoryImpl implements DictRepository { private final DictUtils dictConverter; private final DimensionService dimensionService; - public DictRepositoryImpl(DictTaskMapper dictTaskMapper, DictConfMapper dictConfMapper, - DictUtils dictConverter, DimensionService dimensionService) { + public DictRepositoryImpl( + DictTaskMapper dictTaskMapper, + DictConfMapper dictConfMapper, + DictUtils dictConverter, + DimensionService dimensionService) { this.dictTaskMapper = dictTaskMapper; this.dictConfMapper = dictConfMapper; this.dictConverter = dictConverter; @@ -88,9 +90,11 @@ public class DictRepositoryImpl implements DictRepository { QueryWrapper wrapper = new QueryWrapper<>(); wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId()); wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType()); - List dictTaskDOList = dictTaskMapper.selectList(wrapper).stream() - .sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed()) - .limit(dictTaskNum).collect(Collectors.toList()); + List dictTaskDOList = + dictTaskMapper.selectList(wrapper).stream() + .sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed()) + .limit(dictTaskNum) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dictTaskDOList)) { return taskResp; } @@ -109,10 +113,11 @@ public class DictRepositoryImpl implements DictRepository { @Override public Long editDictConf(DictConfDO dictConfDO) { - DictItemFilter filter = DictItemFilter.builder() - .type(TypeEnums.valueOf(dictConfDO.getType())) - .itemId(dictConfDO.getItemId()) - .build(); + DictItemFilter filter = + DictItemFilter.builder() + .type(TypeEnums.valueOf(dictConfDO.getType())) + .itemId(dictConfDO.getItemId()) + .build(); List dictConfDOList = getDictConfDOList(filter); if (CollectionUtils.isEmpty(dictConfDOList)) { @@ -146,4 +151,4 @@ public class DictRepositoryImpl implements DictRepository { List dictConfDOList = dictConfMapper.selectList(wrapper); return dictConfDOList; } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index 392c5547f..931853c0c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -6,9 +6,10 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapp import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; -import java.util.List; import org.springframework.stereotype.Service; +import java.util.List; + @Service public class DimensionRepositoryImpl implements DimensionRepository { @@ -16,8 +17,8 @@ public class DimensionRepositoryImpl implements DimensionRepository { private DimensionDOCustomMapper dimensionDOCustomMapper; - public DimensionRepositoryImpl(DimensionDOMapper dimensionDOMapper, - DimensionDOCustomMapper dimensionDOCustomMapper) { + public DimensionRepositoryImpl( + DimensionDOMapper dimensionDOMapper, DimensionDOCustomMapper dimensionDOCustomMapper) { this.dimensionDOMapper = dimensionDOMapper; this.dimensionDOCustomMapper = dimensionDOCustomMapper; } @@ -56,5 +57,4 @@ public class DimensionRepositoryImpl implements DimensionRepository { public List getDimensions(DimensionsFilter dimensionsFilter) { return dimensionDOCustomMapper.queryDimensions(dimensionsFilter); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DomainRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DomainRepositoryImpl.java index 98297fa65..30267b57a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DomainRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DomainRepositoryImpl.java @@ -9,7 +9,6 @@ import org.springframework.stereotype.Component; import java.util.List; - @Component @Slf4j public class DomainRepositoryImpl implements DomainRepository { @@ -44,5 +43,4 @@ public class DomainRepositoryImpl implements DomainRepository { public DomainDO getDomainById(Long id) { return domainDOMapper.selectById(id); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java index 975eff973..f565cdadd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java @@ -10,22 +10,22 @@ import com.tencent.supersonic.headless.server.persistence.repository.MetricRepos import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import org.springframework.stereotype.Component; -import java.util.List; +import java.util.List; @Component public class MetricRepositoryImpl implements MetricRepository { - private MetricDOMapper metricDOMapper; private MetricDOCustomMapper metricDOCustomMapper; private MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper; - public MetricRepositoryImpl(MetricDOMapper metricDOMapper, - MetricDOCustomMapper metricDOCustomMapper, - MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper) { + public MetricRepositoryImpl( + MetricDOMapper metricDOMapper, + MetricDOCustomMapper metricDOCustomMapper, + MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper) { this.metricDOMapper = metricDOMapper; this.metricDOCustomMapper = metricDOCustomMapper; this.metricQueryDefaultConfigDOMapper = metricQueryDefaultConfigDOMapper; @@ -95,9 +95,10 @@ public class MetricRepositoryImpl implements MetricRepository { @Override public MetricQueryDefaultConfigDO getDefaultQueryConfig(Long metricId, String userName) { QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(MetricQueryDefaultConfigDO::getMetricId, metricId) + queryWrapper + .lambda() + .eq(MetricQueryDefaultConfigDO::getMetricId, metricId) .eq(MetricQueryDefaultConfigDO::getCreatedBy, userName); return metricQueryDefaultConfigDOMapper.selectOne(queryWrapper); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java index e68a7bc4c..a8b212458 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java @@ -12,17 +12,15 @@ import org.springframework.util.CollectionUtils; import java.util.List; - @Component public class ModelRepositoryImpl implements ModelRepository { - private ModelDOMapper modelDOMapper; private ModelDOCustomMapper modelDOCustomMapper; - public ModelRepositoryImpl(ModelDOMapper modelDOMapper, - ModelDOCustomMapper modelDOCustomMapper) { + public ModelRepositoryImpl( + ModelDOMapper modelDOMapper, ModelDOCustomMapper modelDOCustomMapper) { this.modelDOMapper = modelDOMapper; this.modelDOCustomMapper = modelDOCustomMapper; } @@ -71,5 +69,4 @@ public class ModelRepositoryImpl implements ModelRepository { public void batchUpdate(List modelDOS) { modelDOCustomMapper.batchUpdateStatus(modelDOS); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java index 16955d7a1..62bd35806 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java @@ -57,4 +57,4 @@ public class QueryRuleRepositoryImpl implements QueryRuleRepository { queryRuleDOList.addAll(queryRuleDOListSys); return queryRuleDOList; } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java index 81969f9fa..0ebf9ef31 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java @@ -45,22 +45,34 @@ public class StatRepositoryImpl implements StatRepository { List result = new ArrayList<>(); List statInfos = statMapper.getStatInfo(itemUseReq); Map map = new ConcurrentHashMap<>(); - statInfos.stream().forEach(stat -> { - String dimensions = stat.getDimensions(); - String metrics = stat.getMetrics(); - if (Objects.nonNull(stat.getDataSetId())) { - updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getDataSetId()); - updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getDataSetId()); - } - }); - map.forEach((k, v) -> { - Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]); - String type = k.split(AT_SYMBOL + AT_SYMBOL)[1]; - String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2]; - result.add(new ItemUseResp(classId, type, nameEn, v)); - }); + statInfos.stream() + .forEach( + stat -> { + String dimensions = stat.getDimensions(); + String metrics = stat.getMetrics(); + if (Objects.nonNull(stat.getDataSetId())) { + updateStatMapInfo( + map, + dimensions, + TypeEnums.DIMENSION.name().toLowerCase(), + stat.getDataSetId()); + updateStatMapInfo( + map, + metrics, + TypeEnums.METRIC.name().toLowerCase(), + stat.getDataSetId()); + } + }); + map.forEach( + (k, v) -> { + Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]); + String type = k.split(AT_SYMBOL + AT_SYMBOL)[1]; + String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2]; + result.add(new ItemUseResp(classId, type, nameEn, v)); + }); - return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed()) + return result.stream() + .sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed()) .collect(Collectors.toList()); } @@ -69,19 +81,24 @@ public class StatRepositoryImpl implements StatRepository { return statMapper.getStatInfo(itemUseCommend); } - private void updateStatMapInfo(Map map, String dimensions, String type, Long dataSetId) { + private void updateStatMapInfo( + Map map, String dimensions, String type, Long dataSetId) { if (StringUtils.isNotEmpty(dimensions)) { try { - List dimensionList = mapper.readValue(dimensions, new TypeReference>() { - }); - dimensionList.stream().forEach(dimension -> { - String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension; - if (map.containsKey(key)) { - map.put(key, map.get(key) + 1); - } else { - map.put(key, 1L); - } - }); + List dimensionList = + mapper.readValue(dimensions, new TypeReference>() {}); + dimensionList.stream() + .forEach( + dimension -> { + String key = + dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + + AT_SYMBOL + dimension; + if (map.containsKey(key)) { + map.put(key, map.get(key) + 1); + } else { + map.put(key, 1L); + } + }); } catch (Exception e) { log.warn("e:{}", e); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagObjectRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagObjectRepositoryImpl.java index cb2014140..054384313 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagObjectRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagObjectRepositoryImpl.java @@ -60,4 +60,4 @@ public class TagObjectRepositoryImpl implements TagObjectRepository { } return tagObjectMapper.selectList(wrapper); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java index 3a4627868..6a70e254a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java @@ -20,8 +20,7 @@ public class TagRepositoryImpl implements TagRepository { private final TagMapper mapper; private final TagCustomMapper tagCustomMapper; - public TagRepositoryImpl(TagMapper mapper, - TagCustomMapper tagCustomMapper) { + public TagRepositoryImpl(TagMapper mapper, TagCustomMapper tagCustomMapper) { this.mapper = mapper; this.tagCustomMapper = tagCustomMapper; } @@ -62,8 +61,10 @@ public class TagRepositoryImpl implements TagRepository { if (CollectionUtils.isNotEmpty(tagDeleteReq.getIds())) { tagCustomMapper.deleteBatchByIds(tagDeleteReq.getIds()); } - if (Objects.nonNull(tagDeleteReq.getTagDefineType()) && CollectionUtils.isNotEmpty(tagDeleteReq.getItemIds())) { - tagCustomMapper.deleteBatchByType(tagDeleteReq.getItemIds(), tagDeleteReq.getTagDefineType().name()); + if (Objects.nonNull(tagDeleteReq.getTagDefineType()) + && CollectionUtils.isNotEmpty(tagDeleteReq.getItemIds())) { + tagCustomMapper.deleteBatchByType( + tagDeleteReq.getItemIds(), tagDeleteReq.getTagDefineType().name()); } } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClassFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClassFilter.java index a64c3a2c4..70375330e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClassFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClassFilter.java @@ -9,4 +9,4 @@ public class ClassFilter extends MetaFilter { private String type; private Long dataSetId; private Long classId; -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClickHouseParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClickHouseParametersBuilder.java index 80eda83e1..c2e41355d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClickHouseParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClickHouseParametersBuilder.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.pojo; - import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java index 6825dc1e8..79144b6a8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.pojo; import lombok.Builder; import lombok.Data; + import java.util.List; @Data @@ -11,5 +12,4 @@ public class DataDownload { List> headers; List> data; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatabaseParameter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatabaseParameter.java index 80be20b02..720467cb7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatabaseParameter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatabaseParameter.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.headless.server.pojo; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @Data @AllArgsConstructor @NoArgsConstructor diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java index 3d8b639b5..c82315eb0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.server.pojo; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; + import java.util.LinkedHashMap; import java.util.Map; - public class DbParameterFactory { private static Map parametersBuilder; @@ -25,5 +25,4 @@ public class DbParameterFactory { public static Map getMap() { return parametersBuilder; } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParametersBuilder.java index 8a69ca167..dd89955fe 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParametersBuilder.java @@ -5,5 +5,4 @@ import java.util.List; public interface DbParametersBuilder { List build(); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionFilter.java index cd110b15d..e506de7ce 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionFilter.java @@ -3,10 +3,8 @@ package com.tencent.supersonic.headless.server.pojo; import com.tencent.supersonic.headless.api.pojo.MetaFilter; import lombok.Data; - @Data public class DimensionFilter extends MetaFilter { private Integer isTag; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java index e1e382967..11eb84a8f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.server.pojo; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class DimensionsFilter { @@ -11,5 +12,4 @@ public class DimensionsFilter { private List dimensionIds; private List dimensionNames; - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/H2ParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/H2ParametersBuilder.java index 7aa9638ee..2cd5eabc4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/H2ParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/H2ParametersBuilder.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.server.pojo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; @Service @Slf4j diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java index b8e34f64c..2421d50ed 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.server.pojo; -import java.util.List; - import com.tencent.supersonic.headless.api.pojo.MetaFilter; import lombok.Data; +import java.util.List; + @Data public class MetricsFilter extends MetaFilter { @@ -13,5 +13,4 @@ public class MetricsFilter extends MetaFilter { private List metricIds; private List metricNames; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java index 63dd6aab8..8cd21f8a0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java @@ -21,4 +21,4 @@ public class ModelCluster { modelCluster.setContainsPartitionDimensions(containsPartitionDimensions); return modelCluster; } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java index 941a97983..8c38ccfef 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java @@ -14,9 +14,7 @@ public class ModelFilter extends MetaFilter { private Boolean includesDetail = true; - public ModelFilter() { - - } + public ModelFilter() {} public ModelFilter(Boolean includesDetail) { this.includesDetail = includesDetail; @@ -26,5 +24,4 @@ public class ModelFilter extends MetaFilter { this.includesDetail = includesDetail; this.setIds(ids); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MysqlParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MysqlParametersBuilder.java index 1176c425d..0e653bc6e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MysqlParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MysqlParametersBuilder.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.pojo; - import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java index 4dd4cd79d..1d77e6206 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.pojo; - import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java index 54b05d00b..51f036ca7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.pojo; - import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagObjectFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagObjectFilter.java index 21da39ab1..368c01ca9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagObjectFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagObjectFilter.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.pojo; - import com.tencent.supersonic.headless.api.pojo.MetaFilter; import lombok.Data; @@ -10,4 +9,4 @@ import java.util.List; public class TagObjectFilter extends MetaFilter { private List domainIds; -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java index a4e39f695..67813590c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.server.pojo.yaml; -import com.tencent.supersonic.headless.api.pojo.enums.ModelSourceType; import com.tencent.supersonic.headless.api.pojo.Field; +import com.tencent.supersonic.headless.api.pojo.enums.ModelSourceType; import lombok.Data; import java.util.List; - @Data public class DataModelYamlTpl { @@ -31,7 +30,4 @@ public class DataModelYamlTpl { private List fields; private ModelSourceType modelSourceTypeEnum; - - - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java index 7d4f5084a..b834b3be6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java @@ -2,12 +2,10 @@ package com.tencent.supersonic.headless.server.pojo.yaml; import lombok.Data; - @Data public class DimensionTimeTypeParamsTpl { private String isPrimary; private String timeGranularity; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java index 8f2882634..8d6f0a0c1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java @@ -1,11 +1,10 @@ package com.tencent.supersonic.headless.server.pojo.yaml; - import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; -import java.util.List; -import java.util.Map; import lombok.Data; +import java.util.List; +import java.util.Map; @Data public class DimensionYamlTpl { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/FieldParamYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/FieldParamYamlTpl.java index f701fe54c..b56149832 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/FieldParamYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/FieldParamYamlTpl.java @@ -6,5 +6,4 @@ import lombok.Data; public class FieldParamYamlTpl { private String fieldName; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java index cd612cf04..4cf3793f6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java @@ -4,7 +4,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - @Data @AllArgsConstructor @NoArgsConstructor @@ -12,9 +11,6 @@ public class IdentifyYamlTpl { private String name; - /** - * 主键 primary 外键 foreign - */ + /** 主键 primary 外键 foreign */ private String type; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java index aa797d5c9..914447323 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java @@ -4,7 +4,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - @Data @AllArgsConstructor @NoArgsConstructor @@ -21,5 +20,4 @@ public class MeasureYamlTpl { private String alias; private String createMetric; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricParamYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricParamYamlTpl.java index 2b2cbace3..672d03d1c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricParamYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricParamYamlTpl.java @@ -8,5 +8,4 @@ public class MetricParamYamlTpl { private Long id; private String bizName; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java index 8cec4b7bc..ca160c11c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.pojo.yaml; import com.google.common.collect.Lists; import lombok.Data; + import java.util.List; @Data @@ -14,5 +15,4 @@ public class MetricTypeParamsYamlTpl { private List fields = Lists.newArrayList(); private String expr; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java index d60038ef2..31a83a15e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java @@ -1,11 +1,9 @@ package com.tencent.supersonic.headless.server.pojo.yaml; - import lombok.Data; import java.util.List; - @Data public class MetricYamlTpl { @@ -16,6 +14,4 @@ public class MetricYamlTpl { private String type; private MetricTypeParamsYamlTpl typeParams; - - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java index 3681ed453..024d5aeb3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java @@ -9,23 +9,31 @@ import com.tencent.supersonic.headless.chat.query.QueryManager; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; /** - * EntityInfoProcessor fills core attributes of an entity so that - * users get to know which entity is parsed out. + * EntityInfoProcessor fills core attributes of an entity so that users get to know which entity is + * parsed out. */ public class EntityInfoProcessor implements ResultProcessor { @Override public void process(ParseResp parseResp, ChatQueryContext chatQueryContext) { - parseResp.getSelectedParses().forEach(parseInfo -> { - String queryMode = parseInfo.getQueryMode(); - if (!QueryManager.isDetailQuery(queryMode) && !QueryManager.isMetricQuery(queryMode)) { - return; - } + parseResp + .getSelectedParses() + .forEach( + parseInfo -> { + String queryMode = parseInfo.getQueryMode(); + if (!QueryManager.isDetailQuery(queryMode) + && !QueryManager.isMetricQuery(queryMode)) { + return; + } - SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); - DataSetSchema dataSetSchema = semanticService.getDataSetSchema(parseInfo.getDataSetId()); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, chatQueryContext.getUser()); - parseInfo.setEntityInfo(entityInfo); - }); + SemanticLayerService semanticService = + ContextUtils.getBean(SemanticLayerService.class); + DataSetSchema dataSetSchema = + semanticService.getDataSetSchema(parseInfo.getDataSetId()); + EntityInfo entityInfo = + semanticService.getEntityInfo( + parseInfo, dataSetSchema, chatQueryContext.getUser()); + parseInfo.setEntityInfo(entityInfo); + }); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java index aa73e4cac..1b061c9bc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java @@ -29,10 +29,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -/** - * ParseInfoProcessor extracts structured info from S2SQL so that - * users get to know the details. - **/ +/** ParseInfoProcessor extracts structured info from S2SQL so that users get to know the details. */ @Slf4j public class ParseInfoProcessor implements ResultProcessor { @@ -49,10 +46,11 @@ public class ParseInfoProcessor implements ResultProcessor { } List expressions = SqlSelectHelper.getFilterExpression(s2SQL); Long dataSetId = parseInfo.getDataSetId(); - SemanticLayerService semanticLayerService = ContextUtils.getBean(SemanticLayerService.class); + SemanticLayerService semanticLayerService = + ContextUtils.getBean(SemanticLayerService.class); DataSetSchema dsSchema = semanticLayerService.getDataSetSchema(dataSetId); - //extract date filter from S2SQL + // extract date filter from S2SQL try { if (parseInfo.getDateInfo() == null && !CollectionUtils.isEmpty(expressions)) { parseInfo.setDateInfo(extractDateFilter(expressions, dsSchema)); @@ -61,7 +59,7 @@ public class ParseInfoProcessor implements ResultProcessor { log.error("failed to extract date range:", e); } - //extract dimension filters from S2SQL + // extract dimension filters from S2SQL try { List queryFilters = extractDimensionFilter(dsSchema, expressions); parseInfo.getDimensionFilters().addAll(queryFilters); @@ -69,37 +67,44 @@ public class ParseInfoProcessor implements ResultProcessor { log.error("failed to extract dimension filters:", e); } - //extract metrics from S2SQL - List allFields = filterDateField(dsSchema, SqlSelectHelper.getAllSelectFields(s2SQL)); + // extract metrics from S2SQL + List allFields = + filterDateField(dsSchema, SqlSelectHelper.getAllSelectFields(s2SQL)); Set metrics = matchSchemaElements(allFields, dsSchema.getMetrics()); parseInfo.setMetrics(metrics); - //extract dimensions from S2SQL + // extract dimensions from S2SQL if (QueryType.METRIC.equals(parseInfo.getQueryType())) { List groupByFields = SqlSelectHelper.getGroupByFields(s2SQL); List groupByDimensions = filterDateField(dsSchema, groupByFields); - parseInfo.setDimensions(matchSchemaElements(groupByDimensions, dsSchema.getDimensions())); + parseInfo.setDimensions( + matchSchemaElements(groupByDimensions, dsSchema.getDimensions())); } else if (QueryType.DETAIL.equals(parseInfo.getQueryType())) { List selectFields = SqlSelectHelper.getSelectFields(s2SQL); List selectDimensions = filterDateField(dsSchema, selectFields); - parseInfo.setDimensions(matchSchemaElements(selectDimensions, dsSchema.getDimensions())); + parseInfo.setDimensions( + matchSchemaElements(selectDimensions, dsSchema.getDimensions())); } } - private Set matchSchemaElements(List allFields, Set elements) { + private Set matchSchemaElements( + List allFields, Set elements) { return elements.stream() - .filter(schemaElement -> { + .filter( + schemaElement -> { if (CollectionUtils.isEmpty(schemaElement.getAlias())) { return allFields.contains(schemaElement.getName()); } Set allFieldsSet = new HashSet<>(allFields); Set aliasSet = new HashSet<>(schemaElement.getAlias()); - List intersection = allFieldsSet.stream() - .filter(aliasSet::contains).collect(Collectors.toList()); + List intersection = + allFieldsSet.stream() + .filter(aliasSet::contains) + .collect(Collectors.toList()); return allFields.contains(schemaElement.getName()) || !CollectionUtils.isEmpty(intersection); - } - ).collect(Collectors.toSet()); + }) + .collect(Collectors.toSet()); } private List filterDateField(DataSetSchema dataSetSchema, List allFields) { @@ -108,8 +113,8 @@ public class ParseInfoProcessor implements ResultProcessor { .collect(Collectors.toList()); } - private List extractDimensionFilter(DataSetSchema dsSchema, - List fieldExpressions) { + private List extractDimensionFilter( + DataSetSchema dsSchema, List fieldExpressions) { Map fieldNameToElement = getNameToElement(dsSchema); List result = Lists.newArrayList(); @@ -117,14 +122,16 @@ public class ParseInfoProcessor implements ResultProcessor { QueryFilter dimensionFilter = new QueryFilter(); dimensionFilter.setValue(expression.getFieldValue()); SchemaElement schemaElement = fieldNameToElement.get(expression.getFieldName()); - if (Objects.isNull(schemaElement) || isPartitionDimension(dsSchema, schemaElement.getName())) { + if (Objects.isNull(schemaElement) + || isPartitionDimension(dsSchema, schemaElement.getName())) { continue; } dimensionFilter.setName(schemaElement.getName()); dimensionFilter.setBizName(schemaElement.getBizName()); dimensionFilter.setElementID(schemaElement.getId()); - FilterOperatorEnum operatorEnum = FilterOperatorEnum.getSqlOperator(expression.getOperator()); + FilterOperatorEnum operatorEnum = + FilterOperatorEnum.getSqlOperator(expression.getOperator()); dimensionFilter.setOperator(operatorEnum); dimensionFilter.setFunction(expression.getFunction()); result.add(dimensionFilter); @@ -132,11 +139,15 @@ public class ParseInfoProcessor implements ResultProcessor { return result; } - private DateConf extractDateFilter(List fieldExpressions, - DataSetSchema dataSetSchema) { - List dateExpressions = fieldExpressions.stream() - .filter(expression -> isPartitionDimension(dataSetSchema, expression.getFieldName())) - .collect(Collectors.toList()); + private DateConf extractDateFilter( + List fieldExpressions, DataSetSchema dataSetSchema) { + List dateExpressions = + fieldExpressions.stream() + .filter( + expression -> + isPartitionDimension( + dataSetSchema, expression.getFieldName())) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dateExpressions)) { return null; } @@ -144,21 +155,29 @@ public class ParseInfoProcessor implements ResultProcessor { dateInfo.setDateMode(DateConf.DateMode.BETWEEN); FieldExpression firstExpression = dateExpressions.get(0); - FilterOperatorEnum firstOperator = FilterOperatorEnum.getSqlOperator(firstExpression.getOperator()); - if (FilterOperatorEnum.EQUALS.equals(firstOperator) && Objects.nonNull(firstExpression.getFieldValue())) { + FilterOperatorEnum firstOperator = + FilterOperatorEnum.getSqlOperator(firstExpression.getOperator()); + if (FilterOperatorEnum.EQUALS.equals(firstOperator) + && Objects.nonNull(firstExpression.getFieldValue())) { dateInfo.setStartDate(firstExpression.getFieldValue().toString()); dateInfo.setEndDate(firstExpression.getFieldValue().toString()); dateInfo.setDateMode(DateConf.DateMode.BETWEEN); return dateInfo; } - if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN, + if (containOperators( + firstExpression, + firstOperator, + FilterOperatorEnum.GREATER_THAN, FilterOperatorEnum.GREATER_THAN_EQUALS)) { dateInfo.setStartDate(firstExpression.getFieldValue().toString()); if (hasSecondDate(dateExpressions)) { dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString()); } } - if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN, + if (containOperators( + firstExpression, + firstOperator, + FilterOperatorEnum.MINOR_THAN, FilterOperatorEnum.MINOR_THAN_EQUALS)) { dateInfo.setEndDate(firstExpression.getFieldValue().toString()); if (hasSecondDate(dateExpressions)) { @@ -180,14 +199,17 @@ public class ParseInfoProcessor implements ResultProcessor { return sqlFieldName.equalsIgnoreCase(dataSetSchema.getPartitionDimension().getName()); } - private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator, - FilterOperatorEnum... operatorEnums) { - return (Arrays.asList(operatorEnums).contains(firstOperator) && Objects.nonNull( - expression.getFieldValue())); + private boolean containOperators( + FieldExpression expression, + FilterOperatorEnum firstOperator, + FilterOperatorEnum... operatorEnums) { + return (Arrays.asList(operatorEnums).contains(firstOperator) + && Objects.nonNull(expression.getFieldValue())); } private boolean hasSecondDate(List dateExpressions) { - return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue()); + return dateExpressions.size() > 1 + && Objects.nonNull(dateExpressions.get(1).getFieldValue()); } protected Map getNameToElement(DataSetSchema dsSchema) { @@ -197,21 +219,22 @@ public class ParseInfoProcessor implements ResultProcessor { List allElements = Lists.newArrayList(); allElements.addAll(dimensions); allElements.addAll(metrics); - //support alias + // support alias return allElements.stream() - .flatMap(schemaElement -> { - Set> result = new HashSet<>(); - result.add(Pair.of(schemaElement.getName(), schemaElement)); - List aliasList = schemaElement.getAlias(); - if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) { - for (String alias : aliasList) { - result.add(Pair.of(alias, schemaElement)); - } - } - return result.stream(); - }) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, - (value1, value2) -> value2)); + .flatMap( + schemaElement -> { + Set> result = new HashSet<>(); + result.add(Pair.of(schemaElement.getName(), schemaElement)); + List aliasList = schemaElement.getAlias(); + if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) { + for (String alias : aliasList) { + result.add(Pair.of(alias, schemaElement)); + } + } + return result.stream(); + }) + .collect( + Collectors.toMap( + Pair::getLeft, Pair::getRight, (value1, value2) -> value2)); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java index cc5d82ad2..b6edb0beb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java @@ -3,11 +3,8 @@ package com.tencent.supersonic.headless.server.processor; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.chat.ChatQueryContext; -/** - * A ParseResultProcessor wraps things up before returning results to users in parse stage. - */ +/** A ParseResultProcessor wraps things up before returning results to users in parse stage. */ public interface ResultProcessor { void process(ParseResp parseResp, ChatQueryContext chatQueryContext); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/AppController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/AppController.java index 340120884..d812b1f27 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/AppController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/AppController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -17,68 +20,74 @@ import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; @RestController @RequestMapping("/api/semantic/app") public class AppController { - @Autowired - private AppService appService; + @Autowired private AppService appService; @PostMapping - public boolean save(@RequestBody AppReq app, - HttpServletRequest request, HttpServletResponse response) { + public boolean save( + @RequestBody AppReq app, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); appService.save(app, user); return true; } @PutMapping - public boolean update(@RequestBody AppReq app, - HttpServletRequest request, HttpServletResponse response) { + public boolean update( + @RequestBody AppReq app, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); appService.update(app, user); return true; } @PutMapping("/online/{id}") - public boolean online(@PathVariable("id") Integer id, - HttpServletRequest request, HttpServletResponse response) { + public boolean online( + @PathVariable("id") Integer id, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); appService.online(id, user); return true; } @PutMapping("/offline/{id}") - public boolean offline(@PathVariable("id") Integer id, - HttpServletRequest request, HttpServletResponse response) { + public boolean offline( + @PathVariable("id") Integer id, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); appService.offline(id, user); return true; } @DeleteMapping("/{id}") - public boolean delete(@PathVariable("id") Integer id, - HttpServletRequest request, HttpServletResponse response) { + public boolean delete( + @PathVariable("id") Integer id, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); appService.delete(id, user); return true; } @GetMapping("/{id}") - public AppDetailResp getApp(@PathVariable("id") Integer id, - HttpServletRequest request, HttpServletResponse response) { + public AppDetailResp getApp( + @PathVariable("id") Integer id, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return appService.getApp(id, user); } @PostMapping("/page") - public PageInfo pageApp(@RequestBody AppQueryReq appQueryReq, - HttpServletRequest request, HttpServletResponse response) { + public PageInfo pageApp( + @RequestBody AppQueryReq appQueryReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return appService.pageApp(appQueryReq, user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java index 3b6befbf6..601ac5e44 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -16,20 +18,19 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @RequestMapping("/api/semantic/viewInfo") public class CanvasController { - @Autowired - private CanvasService canvasService; + @Autowired private CanvasService canvasService; @PostMapping("/createOrUpdateViewInfo") - public CanvasDO createOrUpdateCanvas(@RequestBody CanvasReq canvasReq, HttpServletRequest request, - HttpServletResponse response) { + public CanvasDO createOrUpdateCanvas( + @RequestBody CanvasReq canvasReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return canvasService.createOrUpdateCanvas(canvasReq, user); } @@ -45,11 +46,11 @@ public class CanvasController { } @GetMapping("/getDomainSchemaRela/{domainId}") - public List getDomainSchema(@PathVariable("domainId") Long domainId, - HttpServletRequest request, - HttpServletResponse response) { + public List getDomainSchema( + @PathVariable("domainId") Long domainId, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return canvasService.getCanvasSchema(domainId, user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ClassController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ClassController.java index 8b6a81345..3b3aecf71 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ClassController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ClassController.java @@ -1,5 +1,9 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.ClassReq; @@ -15,9 +19,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; import java.util.List; @RestController @@ -39,9 +40,10 @@ public class ClassController { * @return */ @PostMapping("/create") - public ClassResp create(@RequestBody @Valid ClassReq classReq, - HttpServletRequest request, - HttpServletResponse response) { + public ClassResp create( + @RequestBody @Valid ClassReq classReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return classService.create(classReq, user); } @@ -55,9 +57,10 @@ public class ClassController { * @return */ @PutMapping("/update") - public ClassResp update(@RequestBody @Valid ClassReq classReq, - HttpServletRequest request, - HttpServletResponse response) { + public ClassResp update( + @RequestBody @Valid ClassReq classReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return classService.update(classReq, user); } @@ -72,10 +75,12 @@ public class ClassController { * @throws Exception */ @DeleteMapping("delete/{id}/{force}") - public Boolean delete(@PathVariable("id") Long id, - @PathVariable("force") Boolean force, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Boolean delete( + @PathVariable("id") Long id, + @PathVariable("force") Boolean force, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return classService.delete(id, force, user); } @@ -90,11 +95,11 @@ public class ClassController { * @throws Exception */ @GetMapping("delete/{id}/{force}") - public List get(@RequestBody @Valid ClassFilter filter, - HttpServletRequest request, - HttpServletResponse response) { + public List get( + @RequestBody @Valid ClassFilter filter, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return classService.getClassList(filter, user); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CollectController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CollectController.java index 632595168..dcd4eade8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CollectController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CollectController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; @@ -11,17 +14,11 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/*** - * 创建收藏指标的逻辑 - */ +/** * 创建收藏指标的逻辑 */ @RestController @RequestMapping("/api/semantic/collect") public class CollectController { - private CollectService collectService; public CollectController(CollectService collectService) { @@ -29,7 +26,8 @@ public class CollectController { } @PostMapping("/createCollectionIndicators") - public boolean createCollectionIndicators(@RequestBody CollectDO collectDO, + public boolean createCollectionIndicators( + @RequestBody CollectDO collectDO, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -38,19 +36,18 @@ public class CollectController { @Deprecated @DeleteMapping("/deleteCollectionIndicators/{id}") - public boolean deleteCollectionIndicators(@PathVariable Long id, - HttpServletRequest request, - HttpServletResponse response) { + public boolean deleteCollectionIndicators( + @PathVariable Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return collectService.unCollect(user, id); } @PostMapping("/deleteCollectionIndicators") - public boolean deleteCollectionIndicators(@RequestBody CollectDO collectDO, - HttpServletRequest request, - HttpServletResponse response) { + public boolean deleteCollectionIndicators( + @RequestBody CollectDO collectDO, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return collectService.unCollect(user, collectDO); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java index ed4de2a14..da3046010 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java @@ -1,10 +1,13 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DataSetService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.DeleteMapping; @@ -16,27 +19,29 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; + import java.util.List; @RestController @RequestMapping("/api/semantic/dataSet") public class DataSetController { - @Autowired - private DataSetService dataSetService; + @Autowired private DataSetService dataSetService; @PostMapping - public DataSetResp save(@RequestBody DataSetReq dataSetReq, - HttpServletRequest request, HttpServletResponse response) { + public DataSetResp save( + @RequestBody DataSetReq dataSetReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dataSetService.save(dataSetReq, user); } @PutMapping - public DataSetResp update(@RequestBody DataSetReq dataSetReq, - HttpServletRequest request, HttpServletResponse response) { + public DataSetResp update( + @RequestBody DataSetReq dataSetReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dataSetService.update(dataSetReq, user); } @@ -54,11 +59,10 @@ public class DataSetController { } @DeleteMapping("/{id}") - public Boolean delete(@PathVariable("id") Long id, - HttpServletRequest request, HttpServletResponse response) { + public Boolean delete( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); dataSetService.delete(id, user); return true; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index 5c5e62310..a3a05ec90 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.DBColumn; @@ -18,8 +21,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -28,7 +29,6 @@ import java.util.Map; @RequestMapping("/api/semantic/database") public class DatabaseController { - private DatabaseService databaseService; public DatabaseController(DatabaseService databaseService) { @@ -36,7 +36,8 @@ public class DatabaseController { } @PostMapping("/testConnect") - public boolean testConnect(@RequestBody DatabaseReq databaseReq, + public boolean testConnect( + @RequestBody DatabaseReq databaseReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -44,7 +45,8 @@ public class DatabaseController { } @PostMapping("/createOrUpdateDatabase") - public DatabaseResp createOrUpdateDatabase(@RequestBody DatabaseReq databaseReq, + public DatabaseResp createOrUpdateDatabase( + @RequestBody DatabaseReq databaseReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -52,15 +54,15 @@ public class DatabaseController { } @GetMapping("/{id}") - public DatabaseResp getDatabase(@PathVariable("id") Long id, HttpServletRequest request, - HttpServletResponse response) { + public DatabaseResp getDatabase( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return databaseService.getDatabase(id, user); } @GetMapping("/getDatabaseList") - public List getDatabaseList(HttpServletRequest request, - HttpServletResponse response) { + public List getDatabaseList( + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return databaseService.getDatabaseList(user); } @@ -72,7 +74,8 @@ public class DatabaseController { } @PostMapping("/executeSql") - public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq, + public SemanticQueryResp executeSql( + @RequestBody SqlExecuteReq sqlExecuteReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -85,27 +88,31 @@ public class DatabaseController { } @RequestMapping("/getTables") - public List getTables(@RequestParam("databaseId") Long databaseId, - @RequestParam("db") String db) throws SQLException { + public List getTables( + @RequestParam("databaseId") Long databaseId, @RequestParam("db") String db) + throws SQLException { return databaseService.getTables(databaseId, db); } @RequestMapping("/getColumnsByName") - public List getColumnsByName(@RequestParam("databaseId") Long databaseId, @RequestParam("db") String db, - @RequestParam("table") String table) throws SQLException { + public List getColumnsByName( + @RequestParam("databaseId") Long databaseId, + @RequestParam("db") String db, + @RequestParam("table") String table) + throws SQLException { return databaseService.getColumns(databaseId, db, table); } @RequestMapping("/getColumnsBySql") - public List getColumnsBySql(@RequestParam("databaseId") Long databaseId, - @RequestParam("sql") String sql) throws SQLException { + public List getColumnsBySql( + @RequestParam("databaseId") Long databaseId, @RequestParam("sql") String sql) + throws SQLException { return databaseService.getColumns(databaseId, sql); } @GetMapping("/getDatabaseParameters") - public Map> getDatabaseParameters(HttpServletRequest request, - HttpServletResponse response) { + public Map> getDatabaseParameters( + HttpServletRequest request, HttpServletResponse response) { return databaseService.getDatabaseParameters(); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DimensionController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DimensionController.java index ae5793f72..43488ae82 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DimensionController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DimensionController.java @@ -1,11 +1,15 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; @@ -14,7 +18,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.DeleteMapping; @@ -25,21 +28,15 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; - @RestController @RequestMapping("/api/semantic/dimension") public class DimensionController { - @Autowired - private DimensionService dimensionService; - - @Autowired - private SemanticLayerService queryService; + @Autowired private DimensionService dimensionService; + @Autowired private SemanticLayerService queryService; /** * 创建维度 @@ -47,52 +44,60 @@ public class DimensionController { * @param dimensionReq */ @PostMapping("/createDimension") - public DimensionResp createDimension(@RequestBody DimensionReq dimensionReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public DimensionResp createDimension( + @RequestBody DimensionReq dimensionReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return dimensionService.createDimension(dimensionReq, user); } @PostMapping("/updateDimension") - public Boolean updateDimension(@RequestBody DimensionReq dimensionReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Boolean updateDimension( + @RequestBody DimensionReq dimensionReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); dimensionService.updateDimension(dimensionReq, user); return true; } @PostMapping("/batchUpdateStatus") - public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchUpdateStatus( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); dimensionService.batchUpdateStatus(metaBatchReq, user); return true; } @PostMapping("/batchUpdateSensitiveLevel") - public Boolean batchUpdateSensitiveLevel(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchUpdateSensitiveLevel( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); dimensionService.batchUpdateSensitiveLevel(metaBatchReq, user); return true; } @PostMapping("/mockDimensionAlias") - public List mockMetricAlias(@RequestBody DimensionReq dimensionReq, - HttpServletRequest request, - HttpServletResponse response) { + public List mockMetricAlias( + @RequestBody DimensionReq dimensionReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dimensionService.mockAlias(dimensionReq, "dimension", user); } @PostMapping("/mockDimensionValuesAlias") - public List mockDimensionValuesAlias(@RequestBody DimensionReq dimensionReq, - HttpServletRequest request, - HttpServletResponse response) { + public List mockDimensionValuesAlias( + @RequestBody DimensionReq dimensionReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dimensionService.mockDimensionValueAlias(dimensionReq, user); } @@ -110,8 +115,9 @@ public class DimensionController { } @GetMapping("/{modelId}/{dimensionName}") - public DimensionResp getDimensionDescByNameAndId(@PathVariable("modelId") Long modelId, - @PathVariable("dimensionName") String dimensionBizName) { + public DimensionResp getDimensionDescByNameAndId( + @PathVariable("modelId") Long modelId, + @PathVariable("dimensionName") String dimensionBizName) { return dimensionService.getDimension(dimensionBizName, modelId); } @@ -121,17 +127,17 @@ public class DimensionController { } @PostMapping("/queryDimValue") - public SemanticQueryResp queryDimValue(@RequestBody DimensionValueReq dimensionValueReq, - HttpServletRequest request, - HttpServletResponse response) { + public SemanticQueryResp queryDimValue( + @RequestBody DimensionValueReq dimensionValueReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return queryService.queryDimensionValue(dimensionValueReq, user); } @DeleteMapping("deleteDimension/{id}") - public Boolean deleteDimension(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean deleteDimension( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); dimensionService.deleteDimension(id, user); return true; @@ -143,5 +149,4 @@ public class DimensionController { metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); return dimensionService.getDimensions(metaFilter); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DomainController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DomainController.java index 350c300cd..32416c655 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DomainController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DomainController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; @@ -14,13 +17,10 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; - @RestController @RequestMapping("/api/semantic/domain") public class DomainController { @@ -32,17 +32,19 @@ public class DomainController { } @PostMapping("/createDomain") - public DomainResp createDomain(@RequestBody DomainReq domainReq, - HttpServletRequest request, - HttpServletResponse response) { + public DomainResp createDomain( + @RequestBody DomainReq domainReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return domainService.createDomain(domainReq, user); } @PostMapping("/updateDomain") - public DomainResp updateDomain(@RequestBody DomainUpdateReq domainUpdateReq, - HttpServletRequest request, - HttpServletResponse response) { + public DomainResp updateDomain( + @RequestBody DomainUpdateReq domainUpdateReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return domainService.updateDomain(domainUpdateReq, user); } @@ -54,8 +56,8 @@ public class DomainController { } @GetMapping("/getDomainList") - public List getDomainList(HttpServletRequest request, - HttpServletResponse response) { + public List getDomainList( + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return domainService.getDomainListWithAdminAuth(user); } @@ -67,8 +69,9 @@ public class DomainController { @GetMapping("/getDomainListByIds/{domainIds}") public List getDomainListByIds(@PathVariable("domainIds") String domainIds) { - return domainService.getDomainList(Arrays.stream(domainIds.split(",")).map(Long::parseLong) - .collect(Collectors.toList())); + return domainService.getDomainList( + Arrays.stream(domainIds.split(",")) + .map(Long::parseLong) + .collect(Collectors.toList())); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java index 099d223c9..64dcc64ab 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -23,106 +26,93 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; import java.util.List; - @RestController @RequestMapping("/api/semantic/knowledge") public class KnowledgeController { - @Autowired - private DictTaskService taskService; + @Autowired private DictTaskService taskService; - @Autowired - private DictConfService confService; + @Autowired private DictConfService confService; - @Autowired - private MetaEmbeddingTask embeddingTask; + @Autowired private MetaEmbeddingTask embeddingTask; - @Autowired - private DictionaryReloadTask dictionaryReloadTask; + @Autowired private DictionaryReloadTask dictionaryReloadTask; /** - * addDictConf-新增item的字典配置 - * Add configuration information for dictionary entries + * addDictConf-新增item的字典配置 Add configuration information for dictionary entries * * @param dictItemReq */ @PostMapping("/conf") - public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq, - HttpServletRequest request, - HttpServletResponse response) { + public DictItemResp addDictConf( + @RequestBody @Valid DictItemReq dictItemReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return confService.addDictConf(dictItemReq, user); } /** - * editDictConf-编辑item的字典配置 - * Edit configuration information for dictionary entries + * editDictConf-编辑item的字典配置 Edit configuration information for dictionary entries * * @param dictItemReq */ @PutMapping("/conf") - public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq, - HttpServletRequest request, - HttpServletResponse response) { + public DictItemResp editDictConf( + @RequestBody @Valid DictItemReq dictItemReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return confService.editDictConf(dictItemReq, user); } /** - * queryDictConf-查询item的字典配置 - * query configuration information for dictionary entries + * queryDictConf-查询item的字典配置 query configuration information for dictionary entries * * @param filter */ @PostMapping("/conf/query") - public List queryDictConf(@RequestBody @Valid DictItemFilter filter, - HttpServletRequest request, - HttpServletResponse response) { + public List queryDictConf( + @RequestBody @Valid DictItemFilter filter, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return confService.queryDictConf(filter, user); } - /** - * addDictTask-实时导入一个item的字典数据 - * write specific item values to the knowledge base + * addDictTask-实时导入一个item的字典数据 write specific item values to the knowledge base * * @param taskReq */ @PostMapping("/task") - public Long addDictTask(@RequestBody DictSingleTaskReq taskReq, - HttpServletRequest request, - HttpServletResponse response) { + public Long addDictTask( + @RequestBody DictSingleTaskReq taskReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return taskService.addDictTask(taskReq, user); } /** - * deleteDictTask-实时删除某一个item的字典数据 - * remove specific item values from the knowledge base + * deleteDictTask-实时删除某一个item的字典数据 remove specific item values from the knowledge base * * @param taskReq */ @PutMapping("/task/delete") - public Long deleteDictTask(@RequestBody DictSingleTaskReq taskReq, - HttpServletRequest request, - HttpServletResponse response) { + public Long deleteDictTask( + @RequestBody DictSingleTaskReq taskReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return taskService.deleteDictTask(taskReq, user); } - /** - * dailyDictTask-手动离线更新所有字典 - */ + /** dailyDictTask-手动离线更新所有字典 */ @PutMapping("/task/all") - public Boolean dailyDictTask( - HttpServletRequest request, - HttpServletResponse response) { + public Boolean dailyDictTask(HttpServletRequest request, HttpServletResponse response) { return taskService.dailyDictTask(); } @@ -132,9 +122,10 @@ public class KnowledgeController { * @param taskReq */ @PostMapping("/task/search") - public DictTaskResp queryLatestDictTask(@RequestBody DictSingleTaskReq taskReq, - HttpServletRequest request, - HttpServletResponse response) { + public DictTaskResp queryLatestDictTask( + @RequestBody DictSingleTaskReq taskReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return taskService.queryLatestDictTask(taskReq, user); } @@ -157,9 +148,10 @@ public class KnowledgeController { * @param dictValueReq */ @PostMapping("/dict/data") - public PageInfo queryDictValue(@RequestBody @Valid DictValueReq dictValueReq, - HttpServletRequest request, - HttpServletResponse response) { + public PageInfo queryDictValue( + @RequestBody @Valid DictValueReq dictValueReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return taskService.queryDictValue(dictValueReq, user); } @@ -170,17 +162,19 @@ public class KnowledgeController { * @param dictValueReq */ @PostMapping("/dict/file") - public String queryDictFilePath(@RequestBody @Valid DictValueReq dictValueReq, - HttpServletRequest request, - HttpServletResponse response) { + public String queryDictFilePath( + @RequestBody @Valid DictValueReq dictValueReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return taskService.queryDictFilePath(dictValueReq, user); } @PostMapping("/dict/reload") - public boolean reloadKnowledge(@RequestBody @Valid DictValueReq dictValueReq, - HttpServletRequest request, - HttpServletResponse response) { + public boolean reloadKnowledge( + @RequestBody @Valid DictValueReq dictValueReq, + HttpServletRequest request, + HttpServletResponse response) { dictionaryReloadTask.reloadKnowledge(); return true; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java index 65b872a5c..0ecd54156 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; @@ -7,13 +9,13 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.service.MetricService; import org.springframework.web.bind.annotation.DeleteMapping; @@ -24,12 +26,9 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; import java.util.Set; - @RestController @RequestMapping("/api/semantic/metric") public class MetricController { @@ -41,23 +40,28 @@ public class MetricController { } @PostMapping("/createMetric") - public MetricResp createMetric(@RequestBody MetricReq metricReq, + public MetricResp createMetric( + @RequestBody MetricReq metricReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return metricService.createMetric(metricReq, user); } @PostMapping("/updateMetric") - public MetricResp updateMetric(@RequestBody MetricReq metricReq, + public MetricResp updateMetric( + @RequestBody MetricReq metricReq, HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return metricService.updateMetric(metricReq, user); } @PostMapping("/batchUpdateStatus") - public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, + public Boolean batchUpdateStatus( + @RequestBody MetaBatchReq metaBatchReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -66,43 +70,48 @@ public class MetricController { } @PostMapping("/batchPublish") - public Boolean batchPublish(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchPublish( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); metricService.batchPublish(metaBatchReq.getIds(), user); return true; } @PostMapping("/batchUnPublish") - public Boolean batchUnPublish(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchUnPublish( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); metricService.batchUnPublish(metaBatchReq.getIds(), user); return true; } @PostMapping("/batchUpdateClassifications") - public Boolean batchUpdateClassifications(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchUpdateClassifications( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); metricService.batchUpdateClassifications(metaBatchReq, user); return true; } @PostMapping("/batchUpdateSensitiveLevel") - public Boolean batchUpdateSensitiveLevel(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchUpdateSensitiveLevel( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); metricService.batchUpdateSensitiveLevel(metaBatchReq, user); return true; } @PostMapping("/mockMetricAlias") - public List mockMetricAlias(@RequestBody MetricBaseReq metricReq, + public List mockMetricAlias( + @RequestBody MetricBaseReq metricReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -121,7 +130,8 @@ public class MetricController { } @PostMapping("/queryMetric") - public PageInfo queryMetric(@RequestBody PageMetricReq pageMetricReq, + public PageInfo queryMetric( + @RequestBody PageMetricReq pageMetricReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -130,23 +140,22 @@ public class MetricController { @Deprecated @GetMapping("getMetric/{modelId}/{bizName}") - public MetricResp getMetric(@PathVariable("modelId") Long modelId, - @PathVariable("bizName") String bizName) { + public MetricResp getMetric( + @PathVariable("modelId") Long modelId, @PathVariable("bizName") String bizName) { return metricService.getMetric(modelId, bizName); } @GetMapping("getMetric/{id}") - public MetricResp getMetric(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) { + public MetricResp getMetric( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return metricService.getMetric(id, user); } @DeleteMapping("deleteMetric/{id}") - public Boolean deleteMetric(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Boolean deleteMetric( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); metricService.deleteMetric(id, user); return true; @@ -176,7 +185,8 @@ public class MetricController { } @PostMapping("/saveMetricQueryDefaultConfig") - public boolean saveMetricQueryDefaultConfig(@RequestBody MetricQueryDefaultConfig queryDefaultConfig, + public boolean saveMetricQueryDefaultConfig( + @RequestBody MetricQueryDefaultConfig queryDefaultConfig, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -185,11 +195,11 @@ public class MetricController { } @RequestMapping("getMetricQueryDefaultConfig/{metricId}") - public MetricQueryDefaultConfig getMetricQueryDefaultConfig(@PathVariable("metricId") Long metricId, + public MetricQueryDefaultConfig getMetricQueryDefaultConfig( + @PathVariable("metricId") Long metricId, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return metricService.getMetricQueryDefaultConfig(metricId, user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java index 0173e6091..2fae146cc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -21,13 +24,10 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; - @RestController @RequestMapping("/api/semantic/model") public class ModelController { @@ -39,32 +39,42 @@ public class ModelController { } @PostMapping("/createModel") - public Boolean createModel(@RequestBody ModelReq modelReq, - HttpServletRequest request, HttpServletResponse response) throws Exception { + public Boolean createModel( + @RequestBody ModelReq modelReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); modelService.createModel(modelReq, user); return true; } @PostMapping("/updateModel") - public Boolean updateModel(@RequestBody ModelReq modelReq, - HttpServletRequest request, HttpServletResponse response) throws Exception { + public Boolean updateModel( + @RequestBody ModelReq modelReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); modelService.updateModel(modelReq, user); return true; } @DeleteMapping("/deleteModel/{modelId}") - public Boolean deleteModel(@PathVariable("modelId") Long modelId, - HttpServletRequest request, HttpServletResponse response) { + public Boolean deleteModel( + @PathVariable("modelId") Long modelId, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); modelService.deleteModel(modelId, user); return true; } @GetMapping("/getModelList/{domainId}") - public List getModelList(@PathVariable("domainId") Long domainId, - HttpServletRequest request, HttpServletResponse response) { + public List getModelList( + @PathVariable("domainId") Long domainId, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN); } @@ -76,7 +86,10 @@ public class ModelController { @GetMapping("/getModelListByIds/{modelIds}") public List getModelListByIds(@PathVariable("modelIds") String modelIds) { - List ids = Arrays.stream(modelIds.split(",")).map(Long::parseLong).collect(Collectors.toList()); + List ids = + Arrays.stream(modelIds.split(",")) + .map(Long::parseLong) + .collect(Collectors.toList()); ModelFilter modelFilter = new ModelFilter(); modelFilter.setIds(ids); return modelService.getModelList(modelFilter); @@ -93,9 +106,10 @@ public class ModelController { } @PostMapping("/batchUpdateStatus") - public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean batchUpdateStatus( + @RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); modelService.batchUpdateStatus(metaBatchReq, user); return true; @@ -105,5 +119,4 @@ public class ModelController { public UnAvailableItemResp getUnAvailableItem(@RequestBody FieldRemovedReq fieldRemovedReq) { return modelService.getUnAvailableItem(fieldRemovedReq); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelRelaController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelRelaController.java index 68578c894..a512af0ff 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelRelaController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelRelaController.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.rest; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.headless.server.service.ModelRelaService; @@ -20,8 +19,7 @@ import java.util.List; @RequestMapping("/api/semantic/modelRela") public class ModelRelaController { - @Autowired - private ModelRelaService modelRelaService; + @Autowired private ModelRelaService modelRelaService; @PostMapping public boolean save(@RequestBody ModelRela modelRela, User user) { @@ -44,5 +42,4 @@ public class ModelRelaController { public void delete(@PathVariable("id") Long id) { modelRelaService.delete(id); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryRuleController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryRuleController.java index c8f629c3b..69e5106eb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryRuleController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryRuleController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryRuleFilter; @@ -14,8 +17,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @@ -38,9 +39,10 @@ public class QueryRuleController { * @throws Exception */ @PostMapping("/create") - public QueryRuleResp create(@RequestBody @Validated QueryRuleReq queryRuleReq, - HttpServletRequest request, - HttpServletResponse response) { + public QueryRuleResp create( + @RequestBody @Validated QueryRuleReq queryRuleReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return queryRuleService.addQueryRule(queryRuleReq, user); } @@ -55,40 +57,42 @@ public class QueryRuleController { * @throws Exception */ @PostMapping("/update") - public QueryRuleResp update(@RequestBody @Validated QueryRuleReq queryRuleReq, - HttpServletRequest request, - HttpServletResponse response) { + public QueryRuleResp update( + @RequestBody @Validated QueryRuleReq queryRuleReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return queryRuleService.updateQueryRule(queryRuleReq, user); } /** * 删除查询规则 + * * @param id * @param request * @param response * @return */ @DeleteMapping("delete/{id}") - public Boolean delete(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean delete( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return queryRuleService.dropQueryRule(id, user); } /** * 查询规则列表 + * * @param request * @param response * @return */ @PostMapping("query") - public List query(@RequestBody @Validated QueryRuleFilter queryRuleFilter, - HttpServletRequest request, - HttpServletResponse response) { + public List query( + @RequestBody @Validated QueryRuleFilter queryRuleFilter, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return queryRuleService.getQueryRuleList(queryRuleFilter, user); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java index 6a9af62a9..72ebf15a6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.AuthType; @@ -12,31 +15,28 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @RequestMapping("/api/semantic/schema") public class SchemaController { - @Autowired - private SchemaService schemaService; + @Autowired private SchemaService schemaService; @GetMapping("/domain/list") - public List getDomainList(HttpServletRequest request, - HttpServletResponse response) { + public List getDomainList( + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return schemaService.getDomainList(user); } @GetMapping("/model/list") - public List getModelList(@RequestParam("domainId") Long domainId, + public List getModelList( + @RequestParam("domainId") Long domainId, @RequestParam("authType") String authType, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return schemaService.getModelList(user, AuthType.valueOf(authType), domainId); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java index d5f3cfdf7..f1c1b4098 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java @@ -1,5 +1,9 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; + import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; @@ -21,9 +25,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; import java.util.List; @RestController @@ -33,8 +34,7 @@ public class TagController { private final TagMetaService tagMetaService; private final TagQueryService tagQueryService; - public TagController(TagMetaService tagMetaService, - TagQueryService tagQueryService) { + public TagController(TagMetaService tagMetaService, TagQueryService tagQueryService) { this.tagMetaService = tagMetaService; this.tagQueryService = tagQueryService; } @@ -49,9 +49,9 @@ public class TagController { * @throws Exception */ @PostMapping("/create") - public TagResp create(@RequestBody TagReq tagReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public TagResp create( + @RequestBody TagReq tagReq, HttpServletRequest request, HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagMetaService.create(tagReq, user); } @@ -66,9 +66,11 @@ public class TagController { * @throws Exception */ @PostMapping("/create/batch") - public Integer createBatch(@RequestBody @Valid List tagReqList, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Integer createBatch( + @RequestBody @Valid List tagReqList, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagMetaService.createBatch(tagReqList, user); } @@ -83,9 +85,11 @@ public class TagController { * @throws Exception */ @PostMapping("/delete/batch") - public Boolean deleteBatch(@RequestBody @Valid List tagDeleteReqList, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Boolean deleteBatch( + @RequestBody @Valid List tagDeleteReqList, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagMetaService.deleteBatch(tagDeleteReqList, user); } @@ -100,9 +104,8 @@ public class TagController { * @throws Exception */ @DeleteMapping("delete/{id}") - public Boolean delete(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) { + public Boolean delete( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); tagMetaService.delete(id, user); return true; @@ -117,9 +120,8 @@ public class TagController { * @return */ @GetMapping("getTag/{id}") - public TagResp getTag(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) { + public TagResp getTag( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return tagMetaService.getTag(id, user); } @@ -135,7 +137,6 @@ public class TagController { return tagMetaService.getTagDOList(tagFilter); } - /** * 获取标签值分布信息 * @@ -146,9 +147,11 @@ public class TagController { * @throws Exception */ @PostMapping("/value/distribution") - public ItemValueResp queryTagValue(@RequestBody ItemValueReq itemValueReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public ItemValueResp queryTagValue( + @RequestBody ItemValueReq itemValueReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagQueryService.queryTagValue(itemValueReq, user); } @@ -163,11 +166,12 @@ public class TagController { * @throws Exception */ @PostMapping("/queryTag/market") - public PageInfo queryTagMarketPage(@RequestBody TagFilterPageReq tagMarketPageReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public PageInfo queryTagMarketPage( + @RequestBody TagFilterPageReq tagMarketPageReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagMetaService.queryTagMarketPage(tagMarketPageReq, user); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagObjectController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagObjectController.java index 5f1b17fd3..aed1c99b7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagObjectController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagObjectController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.TagObjectReq; @@ -13,8 +16,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @@ -37,9 +38,11 @@ public class TagObjectController { * @throws Exception */ @PostMapping("/create") - public TagObjectResp create(@RequestBody TagObjectReq tagObjectReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public TagObjectResp create( + @RequestBody TagObjectReq tagObjectReq, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagObjectService.create(tagObjectReq, user); } @@ -53,9 +56,10 @@ public class TagObjectController { * @return */ @PostMapping("/update") - public TagObjectResp update(@RequestBody TagObjectReq tagObjectReq, - HttpServletRequest request, - HttpServletResponse response) { + public TagObjectResp update( + @RequestBody TagObjectReq tagObjectReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return tagObjectService.update(tagObjectReq, user); } @@ -70,9 +74,9 @@ public class TagObjectController { * @throws Exception */ @DeleteMapping("delete/{id}") - public Boolean delete(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Boolean delete( + @PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); tagObjectService.delete(id, user, true); return true; @@ -80,6 +84,7 @@ public class TagObjectController { /** * 标签对象-查询 + * * @param filter * @param request * @param response @@ -87,10 +92,12 @@ public class TagObjectController { * @throws Exception */ @PostMapping("/query") - public List queryTagObject(@RequestBody TagObjectFilter filter, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public List queryTagObject( + @RequestBody TagObjectFilter filter, + HttpServletRequest request, + HttpServletResponse response) + throws Exception { User user = UserHolder.findUser(request, response); return tagObjectService.getTagObjects(filter, user); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TermController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TermController.java index 2bd80a5cd..c5762799f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TermController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TermController.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.rest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.TermReq; @@ -15,21 +18,19 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController @RequestMapping("/api/semantic/term") public class TermController { - @Autowired - private TermService termService; + @Autowired private TermService termService; @PostMapping("/saveOrUpdate") - public boolean saveOrUpdate(@RequestBody TermReq termReq, - HttpServletRequest request, - HttpServletResponse response) { + public boolean saveOrUpdate( + @RequestBody TermReq termReq, + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); termService.saveOrUpdate(termReq, user); return true; @@ -45,5 +46,4 @@ public class TermController { termService.delete(id); return true; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/AppService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/AppService.java index 4d576a7c2..c4d9a6a21 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/AppService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/AppService.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.service; - import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.AppQueryReq; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/CollectService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/CollectService.java index 7c6323148..673f9d15b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/CollectService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/CollectService.java @@ -3,12 +3,10 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; + import java.util.List; -/** - * @author yannsu - */ - +/** @author yannsu */ public interface CollectService { Boolean collect(User user, CollectDO collectDO); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java index 556f4b417..b62f858bc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import java.util.List; import java.util.Map; @@ -35,5 +35,4 @@ public interface DataSetService { SemanticQueryReq convert(QueryDataSetReq queryDataSetReq); Long getDataSetIdFromSql(String sql, User user); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index 6868775fa..891db430b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -12,7 +12,6 @@ import java.sql.SQLException; import java.util.List; import java.util.Map; - public interface DatabaseService { SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java index 21347f87c..be64ffbc6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java @@ -7,9 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import java.util.List; -/** - * Make relevant settings for the dictionary - */ +/** Make relevant settings for the dictionary */ public interface DictConfService { DictItemResp addDictConf(DictItemReq itemValueReq, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java index f62f740c2..3f306cbd3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java @@ -7,9 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; -/** - * Manage dictionary tasks - */ +/** Manage dictionary tasks */ public interface DictTaskService { Long addDictTask(DictSingleTaskReq taskReq, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java index c310866a5..dd653d2b7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java @@ -5,12 +5,12 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.DataEvent; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import java.util.List; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DomainService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DomainService.java index a87686df9..e26c5a01d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DomainService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DomainService.java @@ -33,5 +33,4 @@ public interface DomainService { Set getDomainAuthSet(User user, AuthType authTypeEnum); Set getDomainChildren(List domainId); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java index 443bea0cf..3bd0f404a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java @@ -1,16 +1,17 @@ package com.tencent.supersonic.headless.server.service; +import javax.servlet.http.HttpServletResponse; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq; -import javax.servlet.http.HttpServletResponse; public interface DownloadService { - void downloadByStruct(DownloadMetricReq downloadStructReq, - User user, HttpServletResponse response) throws Exception; - - void batchDownload(BatchDownloadReq batchDownloadReq, User user, - HttpServletResponse response) throws Exception; + void downloadByStruct( + DownloadMetricReq downloadStructReq, User user, HttpServletResponse response) + throws Exception; + void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) + throws Exception; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/FlightService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/FlightService.java index 572f9fdbf..4fa7aa2fd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/FlightService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/FlightService.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.server.service; -import java.util.concurrent.ExecutorService; import org.apache.arrow.flight.sql.FlightSqlProducer; +import java.util.concurrent.ExecutorService; + public interface FlightService extends FlightSqlProducer { void setLocation(String host, Integer port); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java index fd5d55e67..3bf9a0d15 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.DataEvent; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq; @@ -14,7 +15,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index 0a0fc8d84..955a68af6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -4,13 +4,13 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import java.util.List; @@ -47,5 +47,4 @@ public interface ModelService { DatabaseResp getDatabaseByModelId(Long modelId); void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryRuleService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryRuleService.java index 2e9a35aef..5cc9b287c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryRuleService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryRuleService.java @@ -18,5 +18,4 @@ public interface QueryRuleService { QueryRuleResp getQueryRuleById(Long id, User user); List getQueryRuleList(QueryRuleFilter queryRuleFilter, User user); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java index ce7fe0b6a..ffc6a34b5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java @@ -8,5 +8,4 @@ import java.util.List; public interface RetrieveService { List retrieve(QueryNLReq queryNLReq); - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index a5c2d0603..e6c0222a0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; @@ -17,7 +18,6 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; @@ -57,11 +57,12 @@ public interface SchemaService { List getDomainDataSetTree(); - void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp, - Map> dimensionYamlMap, - List dataModelYamlTplList, - List metricYamlTplList, - Map modelIdName); + void getSchemaYamlTpl( + SemanticSchemaResp semanticSchemaResp, + Map> dimensionYamlMap, + List dataModelYamlTplList, + List metricYamlTplList, + Map modelIdName); ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagObjectService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagObjectService.java index 1f3e65036..2eef94987 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagObjectService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagObjectService.java @@ -23,5 +23,4 @@ public interface TagObjectService { List getTagObjects(TagObjectFilter filter, User user); Map getAllTagObjectMap(); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TermService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TermService.java index 5e97c219b..f073e3767 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TermService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TermService.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.service; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.TermReq; import com.tencent.supersonic.headless.api.pojo.response.TermResp; @@ -18,5 +17,4 @@ public interface TermService { List getTerms(Long domainId); Map> getTermSets(Set domainIds); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/AppServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/AppServiceImpl.java index 9a4611477..e7d9e9451 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/AppServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/AppServiceImpl.java @@ -10,8 +10,9 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.PageUtils; -import com.tencent.supersonic.headless.api.pojo.enums.AppStatus; import com.tencent.supersonic.headless.api.pojo.AppConfig; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; +import com.tencent.supersonic.headless.api.pojo.enums.AppStatus; import com.tencent.supersonic.headless.api.pojo.request.AppQueryReq; import com.tencent.supersonic.headless.api.pojo.request.AppReq; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; @@ -20,7 +21,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.persistence.dataobject.AppDO; import com.tencent.supersonic.headless.server.persistence.mapper.AppMapper; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.AppService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; @@ -35,7 +35,6 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; - @Service public class AppServiceImpl extends ServiceImpl implements AppService { @@ -45,8 +44,8 @@ public class AppServiceImpl extends ServiceImpl implements App private DimensionService dimensionService; - public AppServiceImpl(AppMapper appMapper, MetricService metricService, - DimensionService dimensionService) { + public AppServiceImpl( + AppMapper appMapper, MetricService metricService, DimensionService dimensionService) { this.appMapper = appMapper; this.metricService = metricService; this.dimensionService = dimensionService; @@ -107,17 +106,20 @@ public class AppServiceImpl extends ServiceImpl implements App @Override public PageInfo pageApp(AppQueryReq appQueryReq, User user) { - PageInfo appDOPageInfo = PageHelper.startPage(appQueryReq.getCurrent(), - appQueryReq.getPageSize()) - .doSelectPageInfo(() -> queryApp(appQueryReq)); + PageInfo appDOPageInfo = + PageHelper.startPage(appQueryReq.getCurrent(), appQueryReq.getPageSize()) + .doSelectPageInfo(() -> queryApp(appQueryReq)); PageInfo appPageInfo = PageUtils.pageInfo2PageInfoVo(appDOPageInfo); - Map metricResps = metricService.getMetrics(new MetaFilter()) - .stream().collect(Collectors.toMap(MetricResp::getId, m -> m)); - Map dimensionResps = dimensionService.getDimensions(new MetaFilter()) - .stream().collect(Collectors.toMap(DimensionResp::getId, m -> m)); - appPageInfo.setList(appDOPageInfo.getList().stream().map(appDO - -> convert(appDO, dimensionResps, metricResps, user)) - .collect(Collectors.toList())); + Map metricResps = + metricService.getMetrics(new MetaFilter()).stream() + .collect(Collectors.toMap(MetricResp::getId, m -> m)); + Map dimensionResps = + dimensionService.getDimensions(new MetaFilter()).stream() + .collect(Collectors.toMap(DimensionResp::getId, m -> m)); + appPageInfo.setList( + appDOPageInfo.getList().stream() + .map(appDO -> convert(appDO, dimensionResps, metricResps, user)) + .collect(Collectors.toList())); return appPageInfo; } @@ -139,10 +141,12 @@ public class AppServiceImpl extends ServiceImpl implements App @Override public AppDetailResp getApp(Integer id, User user) { AppDO appDO = getAppDO(id); - Map metricResps = metricService.getMetrics(new MetaFilter()) - .stream().collect(Collectors.toMap(MetricResp::getId, m -> m)); - Map dimensionResps = dimensionService.getDimensions(new MetaFilter()) - .stream().collect(Collectors.toMap(DimensionResp::getId, m -> m)); + Map metricResps = + metricService.getMetrics(new MetaFilter()).stream() + .collect(Collectors.toMap(MetricResp::getId, m -> m)); + Map dimensionResps = + dimensionService.getDimensions(new MetaFilter()).stream() + .collect(Collectors.toMap(DimensionResp::getId, m -> m)); checkAuth(appDO, user); return convertDetail(appDO, dimensionResps, metricResps); } @@ -175,8 +179,11 @@ public class AppServiceImpl extends ServiceImpl implements App && appDO.getOwner().contains(user.getName()); } - private AppResp convert(AppDO appDO, Map dimensionMap, - Map metricMap, User user) { + private AppResp convert( + AppDO appDO, + Map dimensionMap, + Map metricMap, + User user) { AppResp app = new AppResp(); BeanMapper.mapper(appDO, app); AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class); @@ -191,8 +198,8 @@ public class AppServiceImpl extends ServiceImpl implements App return convertDetail(appDO, new HashMap<>(), new HashMap<>()); } - private AppDetailResp convertDetail(AppDO appDO, Map dimensionMap, - Map metricMap) { + private AppDetailResp convertDetail( + AppDO appDO, Map dimensionMap, Map metricMap) { AppDetailResp app = new AppDetailResp(); BeanMapper.mapper(appDO, app); AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class); @@ -202,24 +209,53 @@ public class AppServiceImpl extends ServiceImpl implements App return app; } - private void fillItemName(AppConfig appConfig, Map dimensionMap, - Map metricMap) { - appConfig.getItems().forEach(metricItem -> { - metricItem.setName(metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getName()); - metricItem.setBizName(metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getBizName()); - metricItem.setCreatedBy(metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getCreatedBy()); - metricItem.getRelateItems().forEach(dimensionItem -> { - dimensionItem.setName(dimensionMap.getOrDefault(dimensionItem.getId(), new DimensionResp()).getName()); - dimensionItem.setBizName(dimensionMap.getOrDefault(dimensionItem.getId(), - new DimensionResp()).getBizName()); - dimensionItem.setCreatedBy(dimensionMap.getOrDefault(dimensionItem.getId(), - new DimensionResp()).getCreatedBy()); - }); - }); + private void fillItemName( + AppConfig appConfig, + Map dimensionMap, + Map metricMap) { + appConfig + .getItems() + .forEach( + metricItem -> { + metricItem.setName( + metricMap + .getOrDefault(metricItem.getId(), new MetricResp()) + .getName()); + metricItem.setBizName( + metricMap + .getOrDefault(metricItem.getId(), new MetricResp()) + .getBizName()); + metricItem.setCreatedBy( + metricMap + .getOrDefault(metricItem.getId(), new MetricResp()) + .getCreatedBy()); + metricItem + .getRelateItems() + .forEach( + dimensionItem -> { + dimensionItem.setName( + dimensionMap + .getOrDefault( + dimensionItem.getId(), + new DimensionResp()) + .getName()); + dimensionItem.setBizName( + dimensionMap + .getOrDefault( + dimensionItem.getId(), + new DimensionResp()) + .getBizName()); + dimensionItem.setCreatedBy( + dimensionMap + .getOrDefault( + dimensionItem.getId(), + new DimensionResp()) + .getCreatedBy()); + }); + }); } private String getUniqueId() { return UUID.randomUUID().toString().replaceAll("_", ""); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java index 554af812e..77010bbe3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.server.service.impl; - import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.CanvasReq; import com.tencent.supersonic.headless.api.pojo.response.CanvasSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; @@ -13,11 +13,10 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; import com.tencent.supersonic.headless.server.persistence.mapper.CanvasDOMapper; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.CanvasService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.CanvasService; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -25,16 +24,14 @@ import org.springframework.stereotype.Service; import java.util.List; @Service -public class CanvasServiceImpl extends ServiceImpl implements CanvasService { +public class CanvasServiceImpl extends ServiceImpl + implements CanvasService { - @Autowired - private ModelService modelService; + @Autowired private ModelService modelService; - @Autowired - private DimensionService dimensionService; + @Autowired private DimensionService dimensionService; - @Autowired - private MetricService metricService; + @Autowired private MetricService metricService; @Override public List getCanvasList(Long domainId) { @@ -46,7 +43,8 @@ public class CanvasServiceImpl extends ServiceImpl imp @Override public List getCanvasSchema(Long domainId, User user) { List canvasSchemaResps = Lists.newArrayList(); - List modelResps = modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN); + List modelResps = + modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN); for (ModelResp modelResp : modelResps) { CanvasSchemaResp canvasSchemaResp = new CanvasSchemaResp(); MetaFilter metaFilter = new MetaFilter(); @@ -83,5 +81,4 @@ public class CanvasServiceImpl extends ServiceImpl imp public void deleteCanvas(Long id) { removeById(id); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ClassServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ClassServiceImpl.java index 44957760c..f2f39569d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ClassServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ClassServiceImpl.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.service.impl; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; @@ -99,7 +98,8 @@ public class ClassServiceImpl implements ClassService { if (user.getName().equalsIgnoreCase(classDO.getCreatedBy()) || user.isSuperAdmin()) { return; } - throw new Exception("delete operation is not supported at the moment. Please contact the admin."); + throw new Exception( + "delete operation is not supported at the moment. Please contact the admin."); } @Override @@ -107,5 +107,4 @@ public class ClassServiceImpl implements ClassService { List classDOList = classRepository.getClassDOList(filter); return converter.convert2RespList(classDOList); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CollectServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CollectServiceImpl.java index dd0b6c272..7e643603e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CollectServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CollectServiceImpl.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.service.impl; +import javax.annotation.Resource; + import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.TypeEnums; @@ -10,17 +12,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -import javax.annotation.Resource; import java.util.List; - @Slf4j @Service public class CollectServiceImpl implements CollectService { public static final String type = "metric"; - @Resource - private CollectMapper collectMapper; + @Resource private CollectMapper collectMapper; @Override public Boolean collect(User user, CollectDO collectReq) { @@ -70,5 +69,4 @@ public class CollectServiceImpl implements CollectService { queryWrapper.lambda().eq(CollectDO::getType, typeEnums.name().toLowerCase()); return collectMapper.selectList(queryWrapper); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java index 8c61285f6..8c20bdd58 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java @@ -5,14 +5,15 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; @@ -27,7 +28,6 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.TagItem; import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; @@ -55,23 +55,16 @@ import java.util.stream.Collectors; @Service @Slf4j -public class DataSetServiceImpl - extends ServiceImpl implements DataSetService { +public class DataSetServiceImpl extends ServiceImpl + implements DataSetService { - @Autowired - private DomainService domainService; + @Autowired private DomainService domainService; - @Lazy - @Autowired - private DimensionService dimensionService; + @Lazy @Autowired private DimensionService dimensionService; - @Lazy - @Autowired - private MetricService metricService; + @Lazy @Autowired private MetricService metricService; - @Lazy - @Autowired - private TagMetaService tagMetaService; + @Lazy @Autowired private TagMetaService tagMetaService; @Override public DataSetResp save(DataSetReq dataSetReq, User user) { @@ -159,10 +152,13 @@ public class DataSetServiceImpl List dataSetFilterByAuth = getDataSetFilterByAuth(dataSetResps, user); dataSetRespSet.addAll(dataSetFilterByAuth); if (domainId != null && domainId > 0) { - dataSetRespSet = dataSetRespSet.stream().filter(modelResp -> - modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet()); + dataSetRespSet = + dataSetRespSet.stream() + .filter(modelResp -> modelResp.getDomainId().equals(domainId)) + .collect(Collectors.toSet()); } - return dataSetRespSet.stream().sorted(Comparator.comparingLong(DataSetResp::getId)) + return dataSetRespSet.stream() + .sorted(Comparator.comparingLong(DataSetResp::getId)) .collect(Collectors.toList()); } @@ -172,33 +168,43 @@ public class DataSetServiceImpl .collect(Collectors.toList()); } - private List getDataSetFilterByDomainAuth(List dataSetResps, User user) { + private List getDataSetFilterByDomainAuth( + List dataSetResps, User user) { Set domainResps = domainService.getDomainAuthSet(user, AuthType.ADMIN); if (CollectionUtils.isEmpty(domainResps)) { return Lists.newArrayList(); } - Set domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet()); - return dataSetResps.stream().filter(dataSetResp -> - domainIds.contains(dataSetResp.getDomainId())).collect(Collectors.toList()); + Set domainIds = + domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet()); + return dataSetResps.stream() + .filter(dataSetResp -> domainIds.contains(dataSetResp.getDomainId())) + .collect(Collectors.toList()); } private DataSetResp convert(DataSetDO dataSetDO) { DataSetResp dataSetResp = new DataSetResp(); BeanMapper.mapper(dataSetDO, dataSetResp); - dataSetResp.setDataSetDetail(JSONObject.parseObject(dataSetDO.getDataSetDetail(), DataSetDetail.class)); + dataSetResp.setDataSetDetail( + JSONObject.parseObject(dataSetDO.getDataSetDetail(), DataSetDetail.class)); if (dataSetDO.getQueryConfig() != null) { - dataSetResp.setQueryConfig(JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class)); + dataSetResp.setQueryConfig( + JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class)); } - dataSetResp.setAdmins(StringUtils.isBlank(dataSetDO.getAdmin()) - ? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdmin().split(","))); - dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) - ? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(","))); + dataSetResp.setAdmins( + StringUtils.isBlank(dataSetDO.getAdmin()) + ? Lists.newArrayList() + : Arrays.asList(dataSetDO.getAdmin().split(","))); + dataSetResp.setAdminOrgs( + StringUtils.isBlank(dataSetDO.getAdminOrg()) + ? Lists.newArrayList() + : Arrays.asList(dataSetDO.getAdminOrg().split(","))); dataSetResp.setTypeEnum(TypeEnums.DATASET); - List dimensionItems = tagMetaService.getTagItems(dataSetResp.dimensionIds(), - TagDefineType.DIMENSION); + List dimensionItems = + tagMetaService.getTagItems(dataSetResp.dimensionIds(), TagDefineType.DIMENSION); dataSetResp.setAllDimensions(dimensionItems); - List metricItems = tagMetaService.getTagItems(dataSetResp.metricIds(), TagDefineType.METRIC); + List metricItems = + tagMetaService.getTagItems(dataSetResp.metricIds(), TagDefineType.METRIC); dataSetResp.setAllMetrics(metricItems); return dataSetResp; } @@ -241,10 +247,13 @@ public class DataSetServiceImpl List dataSetList = getDataSetList(metaFilter); return dataSetList.stream() .flatMap( - dataSetResp -> dataSetResp.getAllModels().stream().map(modelId -> - Pair.of(modelId, dataSetResp.getId()))) - .collect(Collectors.groupingBy(Pair::getLeft, - Collectors.mapping(Pair::getRight, Collectors.toList()))); + dataSetResp -> + dataSetResp.getAllModels().stream() + .map(modelId -> Pair.of(modelId, dataSetResp.getId()))) + .collect( + Collectors.groupingBy( + Pair::getLeft, + Collectors.mapping(Pair::getRight, Collectors.toList()))); } @Override @@ -259,7 +268,8 @@ public class DataSetServiceImpl if (!CollectionUtils.isEmpty(allDimensionIds)) { metaFilter.setIds(allDimensionIds); List dimensionResps = dimensionService.getDimensions(metaFilter); - List duplicateDimensionNames = findDuplicates(dimensionResps, DimensionResp::getName); + List duplicateDimensionNames = + findDuplicates(dimensionResps, DimensionResp::getName); if (!duplicateDimensionNames.isEmpty()) { throw new InvalidArgumentException("存在相同的维度名: " + duplicateDimensionNames); } @@ -277,7 +287,8 @@ public class DataSetServiceImpl private List findDuplicates(List list, Function keyExtractor) { return list.stream() .collect(Collectors.groupingBy(keyExtractor, Collectors.counting())) - .entrySet().stream() + .entrySet() + .stream() .filter(entry -> entry.getValue() > 1) .map(Map.Entry::getKey) .map(Object::toString) @@ -299,5 +310,4 @@ public class DataSetServiceImpl log.info("getDataSetIdFromSql dataSetId:{}", dataSetId); return dataSetId; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 2fbf78c7b..1a0192ccc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -36,18 +36,14 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - @Slf4j @Service public class DatabaseServiceImpl extends ServiceImpl implements DatabaseService { - @Autowired - private SqlUtils sqlUtils; + @Autowired private SqlUtils sqlUtils; - @Lazy - @Autowired - private ModelService datasourceService; + @Lazy @Autowired private ModelService datasourceService; @Override public boolean testConnect(DatabaseReq databaseReq, User user) { @@ -72,25 +68,26 @@ public class DatabaseServiceImpl extends ServiceImpl getDatabaseList(User user) { - List databaseResps = list().stream().map(DatabaseConverter::convert) - .collect(Collectors.toList()); + List databaseResps = + list().stream().map(DatabaseConverter::convert).collect(Collectors.toList()); fillPermission(databaseResps, user); return databaseResps; } private void fillPermission(List databaseResps, User user) { - databaseResps.forEach(databaseResp -> { - if (databaseResp.getAdmins().contains(user.getName()) - || user.getName().equalsIgnoreCase(databaseResp.getCreatedBy()) - || user.isSuperAdmin()) { - databaseResp.setHasPermission(true); - databaseResp.setHasEditPermission(true); - databaseResp.setHasUsePermission(true); - } - if (databaseResp.getViewers().contains(user.getName())) { - databaseResp.setHasUsePermission(true); - } - }); + databaseResps.forEach( + databaseResp -> { + if (databaseResp.getAdmins().contains(user.getName()) + || user.getName().equalsIgnoreCase(databaseResp.getCreatedBy()) + || user.isSuperAdmin()) { + databaseResp.setHasPermission(true); + databaseResp.setHasEditPermission(true); + databaseResp.setHasUsePermission(true); + } + if (databaseResp.getViewers().contains(user.getName())) { + databaseResp.setHasUsePermission(true); + } + }); } @Override @@ -100,8 +97,8 @@ public class DatabaseServiceImpl extends ServiceImpl modelResps = datasourceService.getModelList(modelFilter); if (!CollectionUtils.isEmpty(modelResps)) { - List datasourceNames = modelResps.stream() - .map(ModelResp::getName).collect(Collectors.toList()); + List datasourceNames = + modelResps.stream().map(ModelResp::getName).collect(Collectors.toList()); String message = String.format("该数据库被模型%s使用，无法删除", datasourceNames); throw new RuntimeException(message); } @@ -129,7 +126,9 @@ public class DatabaseServiceImpl extends ServiceImpl> getDatabaseParameters() { - return DbParameterFactory.getMap().entrySet().stream().collect(LinkedHashMap::new, - (map, entry) -> map.put(entry.getKey(), entry.getValue().build()), - LinkedHashMap::putAll); + return DbParameterFactory.getMap().entrySet().stream() + .collect( + LinkedHashMap::new, + (map, entry) -> map.put(entry.getKey(), entry.getValue().build()), + LinkedHashMap::putAll); } private SemanticQueryResp queryWithColumns(String sql, Database database) { @@ -177,7 +178,8 @@ public class DatabaseServiceImpl extends ServiceImpl getColumns(DatabaseResp databaseResp, String db, String table) throws SQLException { + public List getColumns(DatabaseResp databaseResp, String db, String table) + throws SQLException { DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); return engineAdaptor.getColumns(DatabaseConverter.getConnectInfo(databaseResp), db, table); } @@ -204,11 +206,11 @@ public class DatabaseServiceImpl extends ServiceImpl dictItemResp = queryDictConf(filter, user).stream().findFirst(); if (dictItemResp.isPresent()) { return dictItemResp.get(); @@ -54,8 +50,7 @@ public class DictConfServiceImpl implements DictConfService { DictItemFilter filter = DictItemFilter.builder().build(); BeanUtils.copyProperties(itemValueReq, filter); filter.setStatus(null); - Optional dictItemResp = queryDictConf(filter, user).stream() - .findFirst(); + Optional dictItemResp = queryDictConf(filter, user).stream().findFirst(); if (dictItemResp.isPresent()) { return true; } @@ -79,4 +74,4 @@ public class DictConfServiceImpl implements DictConfService { public List queryDictConf(DictItemFilter dictItemFilter, User user) { return dictRepository.queryDictConf(dictItemFilter); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index a2e997d44..9ae156e75 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -31,10 +31,13 @@ public class DictTaskServiceImpl implements DictTaskService { @Value("${dict.flush.enable:true}") private Boolean dictFlushEnable; + @Value("${dict.flush.daily.enable:true}") private Boolean dictFlushDailyEnable; + @Value("${dict.file.type:txt}") private String dictFileType; + private String dimValue = "DimValue_%d_%d"; private final DictRepository dictRepository; @@ -43,11 +46,12 @@ public class DictTaskServiceImpl implements DictTaskService { private final FileHandler fileHandler; private final DictWordService dictWordService; - public DictTaskServiceImpl(DictRepository dictRepository, - DictUtils dictConverter, - DictUtils dictUtils, - FileHandler fileHandler, - DictWordService dictWordService) { + public DictTaskServiceImpl( + DictRepository dictRepository, + DictUtils dictConverter, + DictUtils dictUtils, + FileHandler fileHandler, + DictWordService dictWordService) { this.dictRepository = dictRepository; this.dictConverter = dictConverter; this.dictUtils = dictUtils; @@ -65,7 +69,8 @@ public class DictTaskServiceImpl implements DictTaskService { } private Long handleDictTaskByItemResp(DictItemResp dictItemResp, User user) { - DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.PENDING); + DictTaskDO dictTaskDO = + dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.PENDING); log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); dictRepository.addDictTask(dictTaskDO); Long idInDb = dictTaskDO.getId(); @@ -75,10 +80,11 @@ public class DictTaskServiceImpl implements DictTaskService { } private DictItemResp fetchDictItemResp(DictSingleTaskReq taskReq) { - DictItemFilter dictItemFilter = DictItemFilter.builder() - .itemId(taskReq.getItemId()) - .type(taskReq.getType()) - .build(); + DictItemFilter dictItemFilter = + DictItemFilter.builder() + .itemId(taskReq.getItemId()) + .type(taskReq.getType()) + .build(); List dictItemRespList = dictRepository.queryDictConf(dictItemFilter); if (!CollectionUtils.isEmpty(dictItemRespList)) { return dictItemRespList.get(0); @@ -111,7 +117,6 @@ public class DictTaskServiceImpl implements DictTaskService { } catch (Exception e) { log.error("reloadCustomDictionary error", e); } - } @Override @@ -126,7 +131,8 @@ public class DictTaskServiceImpl implements DictTaskService { log.error("reloadCustomDictionary error", e); } // Add a clear dictionary file record - DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.INITIAL); + DictTaskDO dictTaskDO = + dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.INITIAL); log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); dictRepository.addDictTask(dictTaskDO); return 0L; @@ -153,19 +159,29 @@ public class DictTaskServiceImpl implements DictTaskService { @Override public PageInfo queryDictValue(DictValueReq dictValueReq, User user) { - String fileName = String.format("dic_value_%d_%s_%s", - dictValueReq.getModelId(), dictValueReq.getType().name(), dictValueReq.getItemId()) - + Constants.DOT + dictFileType; - PageInfo dictValueRespList = fileHandler.queryDictValue(fileName, dictValueReq); + String fileName = + String.format( + "dic_value_%d_%s_%s", + dictValueReq.getModelId(), + dictValueReq.getType().name(), + dictValueReq.getItemId()) + + Constants.DOT + + dictFileType; + PageInfo dictValueRespList = + fileHandler.queryDictValue(fileName, dictValueReq); return dictValueRespList; } @Override public String queryDictFilePath(DictValueReq dictValueReq, User user) { - String fileName = String.format("dic_value_%d_%s_%s", - dictValueReq.getModelId(), dictValueReq.getType().name(), dictValueReq.getItemId()) - + Constants.DOT + dictFileType; + String fileName = + String.format( + "dic_value_%d_%s_%s", + dictValueReq.getModelId(), + dictValueReq.getType().name(), + dictValueReq.getItemId()) + + Constants.DOT + + dictFileType; return fileHandler.queryDictFilePath(fileName); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java index 792f6d6e1..86f1c541e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java @@ -18,15 +18,12 @@ import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; - @Service @Slf4j public class DictWordService { - @Autowired - private SchemaService schemaService; - @Autowired - private KnowledgeBaseService knowledgeBaseService; + @Autowired private SchemaService schemaService; + @Autowired private KnowledgeBaseService knowledgeBaseService; private List preDictWords = new ArrayList<>(); @@ -40,7 +37,8 @@ public class DictWordService { long startTime = System.currentTimeMillis(); List dictWords = getAllDictWords(); List preDictWords = getPreDictWords(); - if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) { + if (org.apache.commons.collections.CollectionUtils.isEqualCollection( + dictWords, preDictWords)) { log.debug("Dictionary hasn't been reloaded."); return; } @@ -63,7 +61,8 @@ public class DictWordService { return words; } - private void addWordsByType(DictWordType value, List metas, List natures) { + private void addWordsByType( + DictWordType value, List metas, List natures) { metas = distinct(metas); List natureList = WordBuilderFactory.get(value).getDictWords(metas); log.debug("nature type:{} , nature size:{}", value.name(), natureList.size()); @@ -86,10 +85,10 @@ public class DictWordService { return metas; } return metas.stream() - .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1)) + .collect( + Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1)) .values() .stream() .collect(Collectors.toList()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index de068f584..e79a3fd1b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -17,6 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; @@ -32,8 +33,8 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapp import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelRelaService; @@ -42,7 +43,6 @@ import com.tencent.supersonic.headless.server.service.TagMetaService; import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; -import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -63,7 +63,6 @@ import java.util.stream.Collectors; public class DimensionServiceImpl extends ServiceImpl implements DimensionService { - private DimensionRepository dimensionRepository; private ModelService modelService; @@ -78,17 +77,16 @@ public class DimensionServiceImpl extends ServiceImpl dimensionResps = getDimensions(modelId); - Map bizNameMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); - Map nameMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); - List dimensionToInsert = dimensionReqs.stream() - .filter(dimension -> !bizNameMap.containsKey(dimension.getBizName()) - && !nameMap.containsKey(dimension.getName())) - .collect(Collectors.toList()); + Map bizNameMap = + dimensionResps.stream() + .collect( + Collectors.toMap( + DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); + Map nameMap = + dimensionResps.stream() + .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); + List dimensionToInsert = + dimensionReqs.stream() + .filter( + dimension -> + !bizNameMap.containsKey(dimension.getBizName()) + && !nameMap.containsKey(dimension.getName())) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dimensionToInsert)) { return; } - List dimensionDOS = dimensionToInsert.stream().peek(dimension -> - dimension.createdBy(user.getName())) - .map(DimensionConverter::convert2DimensionDO) - .collect(Collectors.toList()); + List dimensionDOS = + dimensionToInsert.stream() + .peek(dimension -> dimension.createdBy(user.getName())) + .map(DimensionConverter::convert2DimensionDO) + .collect(Collectors.toList()); dimensionRepository.createDimensionBatch(dimensionDOS); sendEventBatch(dimensionDOS, EventType.ADD); } @@ -143,9 +149,15 @@ public class DimensionServiceImpl extends ServiceImpl { - dimensionDO.setStatus(metaBatchReq.getStatus()); - dimensionDO.setUpdatedAt(new Date()); - dimensionDO.setUpdatedBy(user.getName()); - }) - .collect(Collectors.toList()); + dimensionDOS = + dimensionDOS.stream() + .peek( + dimensionDO -> { + dimensionDO.setStatus(metaBatchReq.getStatus()); + dimensionDO.setUpdatedAt(new Date()); + dimensionDO.setUpdatedBy(user.getName()); + }) + .collect(Collectors.toList()); dimensionRepository.batchUpdateStatus(dimensionDOS); if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus()) || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) { @@ -233,9 +247,9 @@ public class DimensionServiceImpl extends ServiceImpl dimensionDOPageInfo = PageHelper.startPage(pageDimensionReq.getCurrent(), - pageDimensionReq.getPageSize()) - .doSelectPageInfo(() -> queryDimension(dimensionFilter)); + PageInfo dimensionDOPageInfo = + PageHelper.startPage(pageDimensionReq.getCurrent(), pageDimensionReq.getPageSize()) + .doSelectPageInfo(() -> queryDimension(dimensionFilter)); PageInfo pageInfo = new PageInfo<>(); BeanUtils.copyProperties(dimensionDOPageInfo, pageInfo); pageInfo.setList(convertList(dimensionDOPageInfo.getList())); @@ -273,7 +287,8 @@ public class DimensionServiceImpl extends ServiceImpl filterByField(List dimensionResps, List fields) { + private List filterByField( + List dimensionResps, List fields) { List dimensionFiltered = Lists.newArrayList(); for (DimensionResp dimensionResp : dimensionResps) { for (String field : fields) { @@ -301,27 +316,34 @@ public class DimensionServiceImpl extends ServiceImpl convertList(List dimensionDOS) { - List modelIds = dimensionDOS.stream().map(DimensionDO::getModelId) - .collect(Collectors.toList()); + List modelIds = + dimensionDOS.stream().map(DimensionDO::getModelId).collect(Collectors.toList()); ModelFilter modelFilter = new ModelFilter(false, modelIds); Map modelMap = modelService.getModelMap(modelFilter); List dimensionResps = Lists.newArrayList(); if (!CollectionUtils.isEmpty(dimensionDOS)) { - dimensionResps = dimensionDOS.stream() - .map(dimensionDO -> DimensionConverter - .convert2DimensionResp(dimensionDO, modelMap)) - .collect(Collectors.toList()); + dimensionResps = + dimensionDOS.stream() + .map( + dimensionDO -> + DimensionConverter.convert2DimensionResp( + dimensionDO, modelMap)) + .collect(Collectors.toList()); } return dimensionResps; } @Override public List mockAlias(DimensionReq dimensionReq, String mockType, User user) { - String mockAlias = aliasGenerateHelper.generateAlias(mockType, dimensionReq.getName(), - dimensionReq.getBizName(), "", dimensionReq.getDescription()); + String mockAlias = + aliasGenerateHelper.generateAlias( + mockType, + dimensionReq.getName(), + dimensionReq.getBizName(), + "", + dimensionReq.getDescription()); String ret = aliasGenerateHelper.extractJsonStringFromAiMessage(mockAlias); - return JSONObject.parseObject(ret, new TypeReference>() { - }); + return JSONObject.parseObject(ret, new TypeReference>() {}); } @Override @@ -335,8 +357,13 @@ public class DimensionServiceImpl extends ServiceImpl> resultList = semanticQueryResp.getResultList(); List valueList = new ArrayList<>(); @@ -360,8 +387,11 @@ public class DimensionServiceImpl extends ServiceImpl dimensionReqs) { Long modelId = dimensionReqs.get(0).getModelId(); List dimensionResps = getDimensions(modelId); - Map bizNameMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); - Map nameMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); + Map bizNameMap = + dimensionResps.stream() + .collect( + Collectors.toMap( + DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); + Map nameMap = + dimensionResps.stream() + .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); for (DimensionReq dimensionReq : dimensionReqs) { - String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dimensionReq.getName()); + String forbiddenCharacters = + NameCheckUtils.findForbiddenCharacters(dimensionReq.getName()); if (StringUtils.isNotBlank(forbiddenCharacters)) { - throw new InvalidArgumentException(String.format("名称包含特殊字符, 请修改: %s，特殊字符: %s", - dimensionReq.getName(), forbiddenCharacters)); + throw new InvalidArgumentException( + String.format( + "名称包含特殊字符, 请修改: %s，特殊字符: %s", + dimensionReq.getName(), forbiddenCharacters)); } if (bizNameMap.containsKey(dimensionReq.getBizName())) { DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName()); if (!dimensionResp.getId().equals(dimensionReq.getId())) { - throw new RuntimeException(String.format("该主题域下存在相同的维度字段名:%s 创建人:%s", - dimensionReq.getBizName(), dimensionResp.getCreatedBy())); + throw new RuntimeException( + String.format( + "该主题域下存在相同的维度字段名:%s 创建人:%s", + dimensionReq.getBizName(), dimensionResp.getCreatedBy())); } } if (nameMap.containsKey(dimensionReq.getName())) { DimensionResp dimensionResp = nameMap.get(dimensionReq.getName()); if (!dimensionResp.getId().equals(dimensionReq.getId())) { - throw new RuntimeException(String.format("该主题域下存在相同的维度名:%s 创建人:%s", - dimensionReq.getName(), dimensionResp.getCreatedBy())); + throw new RuntimeException( + String.format( + "该主题域下存在相同的维度名:%s 创建人:%s", + dimensionReq.getName(), dimensionResp.getCreatedBy())); } } } @@ -421,16 +462,23 @@ public class DimensionServiceImpl extends ServiceImpl dimensionDOS, EventType eventType) { - List dataItems = dimensionDOS.stream() - .map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE) - .name(dimensionDO.getName()).modelId(dimensionDO.getModelId() + Constants.UNDERLINE) - .type(TypeEnums.DIMENSION).build()) - .collect(Collectors.toList()); + List dataItems = + dimensionDOS.stream() + .map( + dimensionDO -> + DataItem.builder() + .id(dimensionDO.getId() + Constants.UNDERLINE) + .name(dimensionDO.getName()) + .modelId( + dimensionDO.getModelId() + + Constants.UNDERLINE) + .type(TypeEnums.DIMENSION) + .build()) + .collect(Collectors.toList()); return new DataEvent(this, dataItems, eventType); } private void sendEvent(DataItem dataItem, EventType eventType) { - eventPublisher.publishEvent(new DataEvent(this, - Lists.newArrayList(dataItem), eventType)); + eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType)); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java index f7be8f345..485ef99a9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java @@ -33,7 +33,6 @@ import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; - @Service @Slf4j public class DomainServiceImpl implements DomainService { @@ -42,9 +41,10 @@ public class DomainServiceImpl implements DomainService { private final ModelService modelService; private final UserService userService; - public DomainServiceImpl(DomainRepository domainRepository, - @Lazy ModelService modelService, - UserService userService) { + public DomainServiceImpl( + DomainRepository domainRepository, + @Lazy ModelService modelService, + UserService userService) { this.domainRepository = domainRepository; this.modelService = modelService; this.userService = userService; @@ -98,11 +98,13 @@ public class DomainServiceImpl implements DomainService { public List getDomainListWithAdminAuth(User user) { Set domainWithAuthAll = getDomainAuthSet(user, AuthType.ADMIN); if (!CollectionUtils.isEmpty(domainWithAuthAll)) { - List domainIds = domainWithAuthAll.stream().map(DomainResp::getId).collect(Collectors.toList()); + List domainIds = + domainWithAuthAll.stream().map(DomainResp::getId).collect(Collectors.toList()); domainWithAuthAll.addAll(getParentDomain(domainIds)); } List modelResps = modelService.getModelAuthList(user, null, AuthType.ADMIN); - List domainIdsFromModel = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList()); + List domainIdsFromModel = + modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList()); if (!CollectionUtils.isEmpty(modelResps)) { domainWithAuthAll.addAll(getParentDomain(domainIdsFromModel)); } @@ -111,8 +113,10 @@ public class DomainServiceImpl implements DomainService { domainResp.setHasModel(true); } } - return new ArrayList<>(domainWithAuthAll).stream() - .sorted(Comparator.comparingLong(DomainResp::getId)).collect(Collectors.toList()); + return new ArrayList<>(domainWithAuthAll) + .stream() + .sorted(Comparator.comparingLong(DomainResp::getId)) + .collect(Collectors.toList()); } @Override @@ -121,17 +125,19 @@ public class DomainServiceImpl implements DomainService { Set orgIds = userService.getUserAllOrgId(user.getName()); Set domainWithAuth = Sets.newHashSet(); if (authTypeEnum.equals(AuthType.ADMIN)) { - domainWithAuth = domainResps.stream() - .filter(domainResp -> checkAdminPermission(orgIds, user, domainResp)) - .collect(Collectors.toSet()); + domainWithAuth = + domainResps.stream() + .filter(domainResp -> checkAdminPermission(orgIds, user, domainResp)) + .collect(Collectors.toSet()); return domainWithAuth.stream() .peek(domainResp -> domainResp.setHasEditPermission(true)) .collect(Collectors.toSet()); } if (authTypeEnum.equals(AuthType.VISIBLE)) { - domainWithAuth = domainResps.stream() - .filter(domainResp -> checkViewPermission(orgIds, user, domainResp)) - .collect(Collectors.toSet()); + domainWithAuth = + domainResps.stream() + .filter(domainResp -> checkViewPermission(orgIds, user, domainResp)) + .collect(Collectors.toSet()); } return domainWithAuth; @@ -177,7 +183,8 @@ public class DomainServiceImpl implements DomainService { @Override public Map getDomainMap() { - return getDomainList().stream().collect(Collectors.toMap(DomainResp::getId, a -> a, (k1, k2) -> k1)); + return getDomainList().stream() + .collect(Collectors.toMap(DomainResp::getId, a -> a, (k1, k2) -> k1)); } @Override @@ -210,8 +217,9 @@ public class DomainServiceImpl implements DomainService { public Map getDomainFullPathMap() { Map domainFullPathMap = new HashMap<>(); List domainDOList = domainRepository.getDomainList(); - Map domainDOMap = domainDOList.stream() - .collect(Collectors.toMap(DomainDO::getId, a -> a, (k1, k2) -> k1)); + Map domainDOMap = + domainDOList.stream() + .collect(Collectors.toMap(DomainDO::getId, a -> a, (k1, k2) -> k1)); for (DomainDO domainDO : domainDOList) { final Long domainId = domainDO.getId(); StringBuilder fullPath = new StringBuilder(domainDO.getBizName() + "/"); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index 2d8608c5a..bf3b58837 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.service.impl; +import javax.servlet.http.HttpServletResponse; + import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.util.FileUtils; @@ -13,6 +15,7 @@ import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq; @@ -23,17 +26,15 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.utils.DataTransformUtils; +import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.pojo.DataDownload; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DownloadService; import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; -import javax.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; @@ -51,7 +52,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; - @Slf4j @Service public class DownloadServiceImpl implements DownloadService { @@ -66,25 +66,37 @@ public class DownloadServiceImpl implements DownloadService { private SemanticLayerService queryService; - public DownloadServiceImpl(MetricService metricService, - DimensionService dimensionService, SemanticLayerService queryService) { + public DownloadServiceImpl( + MetricService metricService, + DimensionService dimensionService, + SemanticLayerService queryService) { this.metricService = metricService; this.dimensionService = dimensionService; this.queryService = queryService; } @Override - public void downloadByStruct(DownloadMetricReq downloadMetricReq, - User user, HttpServletResponse response) throws Exception { - String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); + public void downloadByStruct( + DownloadMetricReq downloadMetricReq, User user, HttpServletResponse response) + throws Exception { + String fileName = + String.format( + "%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); try { QueryStructReq queryStructReq = metricService.convert(downloadMetricReq); - SemanticQueryResp queryResult = queryService.queryByReq(queryStructReq.convert(true), user); - DataDownload dataDownload = buildDataDownload(queryResult, queryStructReq, downloadMetricReq.isTransform()); - EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData()); + SemanticQueryResp queryResult = + queryService.queryByReq(queryStructReq.convert(true), user); + DataDownload dataDownload = + buildDataDownload(queryResult, queryStructReq, downloadMetricReq.isTransform()); + EasyExcel.write(file) + .sheet("Sheet1") + .head(dataDownload.getHeaders()) + .doWrite(dataDownload.getData()); } catch (RuntimeException e) { - EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead()) + EasyExcel.write(file) + .sheet("Sheet1") + .head(buildErrMessageHead()) .doWrite(buildErrMessageData(e.getMessage())); return; } @@ -92,9 +104,12 @@ public class DownloadServiceImpl implements DownloadService { } @Override - public void batchDownload(BatchDownloadReq batchDownloadReq, User user, - HttpServletResponse response) throws Exception { - String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); + public void batchDownload( + BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) + throws Exception { + String fileName = + String.format( + "%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); List metricIds = batchDownloadReq.getMetricIds(); if (CollectionUtils.isEmpty(metricIds)) { @@ -104,19 +119,23 @@ public class DownloadServiceImpl implements DownloadService { downloadFile(response, file, fileName); } - public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) throws Exception { + public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) + throws Exception { List metricIds = batchDownloadReq.getMetricIds(); MetaFilter metaFilter = new MetaFilter(); metaFilter.setIds(metricIds); List metricResps = metricService.getMetrics(metaFilter); Map> metricMap = getMetricMap(metricResps); - List dimensionIds = metricResps.stream() - .map(metricResp -> metricService.getDrillDownDimension(metricResp.getId())) - .flatMap(Collection::stream) - .map(DrillDownDimension::getDimensionId).collect(Collectors.toList()); + List dimensionIds = + metricResps.stream() + .map(metricResp -> metricService.getDrillDownDimension(metricResp.getId())) + .flatMap(Collection::stream) + .map(DrillDownDimension::getDimensionId) + .collect(Collectors.toList()); metaFilter.setIds(dimensionIds); - Map dimensionRespMap = dimensionService.getDimensions(metaFilter) - .stream().collect(Collectors.toMap(DimensionResp::getId, d -> d)); + Map dimensionRespMap = + dimensionService.getDimensions(metaFilter).stream() + .collect(Collectors.toMap(DimensionResp::getId, d -> d)); ExcelWriter excelWriter = EasyExcel.write(file).build(); int sheetCount = 1; for (List metrics : metricMap.values()) { @@ -127,17 +146,23 @@ public class DownloadServiceImpl implements DownloadService { List dimensions = getMetricRelaDimensions(metricResp, dimensionRespMap); for (MetricResp metric : metrics) { try { - QueryStructReq queryStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq); + QueryStructReq queryStructReq = + buildDownloadReq(dimensions, metric, batchDownloadReq); QuerySqlReq querySqlReq = queryStructReq.convert(); querySqlReq.setNeedAuth(true); SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); - DataDownload dataDownload = buildDataDownload(queryResult, - queryStructReq, batchDownloadReq.isTransform()); - WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount) - .head(dataDownload.getHeaders()).build(); + DataDownload dataDownload = + buildDataDownload( + queryResult, queryStructReq, batchDownloadReq.isTransform()); + WriteSheet writeSheet = + EasyExcel.writerSheet("Sheet" + sheetCount) + .head(dataDownload.getHeaders()) + .build(); excelWriter.write(dataDownload.getData(), writeSheet); } catch (RuntimeException e) { - EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead()) + EasyExcel.write(file) + .sheet("Sheet1") + .head(buildErrMessageHead()) .doWrite(buildErrMessageData(e.getMessage())); return; } @@ -194,8 +219,11 @@ public class DownloadServiceImpl implements DownloadService { return data; } - private List> buildData(List> headers, Map nameMap, - List> dataTransformed, String metricName) { + private List> buildData( + List> headers, + Map nameMap, + List> dataTransformed, + String metricName) { List> data = Lists.newArrayList(); for (Map map : dataTransformed) { List row = Lists.newArrayList(); @@ -217,18 +245,27 @@ public class DownloadServiceImpl implements DownloadService { return data; } - private DataDownload buildDataDownload(SemanticQueryResp queryResult, - QueryStructReq queryStructReq, boolean isTransform) { + private DataDownload buildDataDownload( + SemanticQueryResp queryResult, QueryStructReq queryStructReq, boolean isTransform) { List metricColumns = queryResult.getMetricColumns(); List dimensionColumns = queryResult.getDimensionColumns(); if (isTransform && !CollectionUtils.isEmpty(metricColumns)) { QueryColumn metric = metricColumns.get(0); List groups = queryStructReq.getGroups(); - List> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), - metric.getNameEn(), groups, queryStructReq.getDateInfo()); - List> headers = buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList()); - List> data = buildData(headers, getDimensionNameMap(dimensionColumns), - dataTransformed, metric.getName()); + List> dataTransformed = + DataTransformUtils.transform( + queryResult.getResultList(), + metric.getNameEn(), + groups, + queryStructReq.getDateInfo()); + List> headers = + buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList()); + List> data = + buildData( + headers, + getDimensionNameMap(dimensionColumns), + dataTransformed, + metric.getName()); return DataDownload.builder().headers(headers).data(data).build(); } else { List> data = buildData(queryResult); @@ -237,14 +274,19 @@ public class DownloadServiceImpl implements DownloadService { } } - private QueryStructReq buildDownloadReq(List dimensionResps, MetricResp metricResp, - BatchDownloadReq batchDownloadReq) { + private QueryStructReq buildDownloadReq( + List dimensionResps, + MetricResp metricResp, + BatchDownloadReq batchDownloadReq) { DateConf dateConf = batchDownloadReq.getDateInfo(); - Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); + Set modelIds = + dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); modelIds.add(metricResp.getModelId()); QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setGroups(dimensionResps.stream() - .map(DimensionResp::getBizName).collect(Collectors.toList())); + queryStructReq.setGroups( + dimensionResps.stream() + .map(DimensionResp::getBizName) + .collect(Collectors.toList())); queryStructReq.getGroups().add(0, getTimeDimension(dateConf)); Aggregator aggregator = new Aggregator(); aggregator.setColumn(metricResp.getBizName()); @@ -267,26 +309,32 @@ public class DownloadServiceImpl implements DownloadService { private Map> getMetricMap(List metricResps) { for (MetricResp metricResp : metricResps) { - List drillDownDimensions = metricService.getDrillDownDimension(metricResp.getId()); - RelateDimension relateDimension = RelateDimension.builder() - .drillDownDimensions(drillDownDimensions).build(); + List drillDownDimensions = + metricService.getDrillDownDimension(metricResp.getId()); + RelateDimension relateDimension = + RelateDimension.builder().drillDownDimensions(drillDownDimensions).build(); metricResp.setRelateDimension(relateDimension); } - return metricResps.stream().collect(Collectors.groupingBy(MetricResp::getRelaDimensionIdKey)); + return metricResps.stream() + .collect(Collectors.groupingBy(MetricResp::getRelaDimensionIdKey)); } private Map getDimensionNameMap(List queryColumns) { - return queryColumns.stream().collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn)); + return queryColumns.stream() + .collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn)); } - private List getMetricRelaDimensions(MetricResp metricResp, - Map dimensionRespMap) { + private List getMetricRelaDimensions( + MetricResp metricResp, Map dimensionRespMap) { if (metricResp.getRelateDimension() == null - || CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { + || CollectionUtils.isEmpty( + metricResp.getRelateDimension().getDrillDownDimensions())) { return Lists.newArrayList(); } - return metricResp.getRelateDimension().getDrillDownDimensions() - .stream().map(drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId())) + return metricResp.getRelateDimension().getDrillDownDimensions().stream() + .map( + drillDownDimension -> + dimensionRespMap.get(drillDownDimension.getDimensionId())) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -296,7 +344,9 @@ public class DownloadServiceImpl implements DownloadService { byte[] buffer = readFileToByteArray(file); response.reset(); response.setCharacterEncoding("UTF-8"); - response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8")); + response.addHeader( + "Content-Disposition", + "attachment;filename=" + URLEncoder.encode(filename, "UTF-8")); response.addHeader("Content-Length", "" + file.length()); try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) { response.setContentType("application/octet-stream"); @@ -315,7 +365,4 @@ public class DownloadServiceImpl implements DownloadService { return buffer; } } - } - - diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/FlightServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/FlightServiceImpl.java index 54509e759..a4421c60f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/FlightServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/FlightServiceImpl.java @@ -1,11 +1,10 @@ package com.tencent.supersonic.headless.server.service.impl; -import static com.google.protobuf.Any.pack; -import static com.google.protobuf.ByteString.copyFrom; -import static java.util.Collections.singletonList; -import static java.util.UUID.randomUUID; -import static org.apache.arrow.adapter.jdbc.JdbcToArrow.sqlToArrowVectorIterator; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowUtils.jdbcToArrowSchema; +import javax.sql.RowSetMetaData; +import javax.sql.rowset.CachedRowSet; +import javax.sql.rowset.RowSetFactory; +import javax.sql.rowset.RowSetMetaDataImpl; +import javax.sql.rowset.RowSetProvider; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -20,27 +19,9 @@ import com.tencent.supersonic.headless.api.pojo.Param; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.server.service.FlightService; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; +import com.tencent.supersonic.headless.server.service.FlightService; import com.tencent.supersonic.headless.server.utils.FlightUtils; -import java.nio.charset.StandardCharsets; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Types; -import java.util.Arrays; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import javax.sql.RowSetMetaData; -import javax.sql.rowset.CachedRowSet; -import javax.sql.rowset.RowSetFactory; -import javax.sql.rowset.RowSetMetaDataImpl; -import javax.sql.rowset.RowSetProvider; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; import org.apache.arrow.adapter.jdbc.JdbcToArrowUtils; @@ -69,9 +50,28 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -/** - * arrow flight FlightSqlProducer - */ +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import static com.google.protobuf.Any.pack; +import static com.google.protobuf.ByteString.copyFrom; +import static java.util.Collections.singletonList; +import static java.util.UUID.randomUUID; +import static org.apache.arrow.adapter.jdbc.JdbcToArrow.sqlToArrowVectorIterator; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowUtils.jdbcToArrowSchema; + +/** arrow flight FlightSqlProducer */ @Slf4j @Service("FlightService") public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightService { @@ -88,9 +88,10 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS private final AuthenticationConfig authenticationConfig; private final UserService userService; - public FlightServiceImpl(SemanticLayerService queryService, - AuthenticationConfig authenticationConfig, - UserService userService) { + public FlightServiceImpl( + SemanticLayerService queryService, + AuthenticationConfig authenticationConfig, + UserService userService) { this.queryService = queryService; this.authenticationConfig = authenticationConfig; @@ -103,7 +104,8 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS } @Override - public void setExecutorService(ExecutorService executorService, Integer queue, Integer expireMinute) { + public void setExecutorService( + ExecutorService executorService, Integer queue, Integer expireMinute) { this.executorService = executorService; this.preparedStatementCache = CacheBuilder.newBuilder() @@ -118,7 +120,9 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS } @Override - public void getStreamStatement(final TicketStatementQuery ticketStatementQuery, final CallContext context, + public void getStreamStatement( + final TicketStatementQuery ticketStatementQuery, + final CallContext context, final ServerStreamListener listener) { final ByteString handle = ticketStatementQuery.getStatementHandle(); log.info("getStreamStatement {} ", handle); @@ -126,13 +130,16 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS } @Override - public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, final CallContext context, + public FlightInfo getFlightInfoStatement( + final CommandStatementQuery request, + final CallContext context, final FlightDescriptor descriptor) { try { ByteString preparedStatementHandle = addPrepared(context, request.getQuery()); - TicketStatementQuery ticket = TicketStatementQuery.newBuilder() - .setStatementHandle(preparedStatementHandle) - .build(); + TicketStatementQuery ticket = + TicketStatementQuery.newBuilder() + .setStatementHandle(preparedStatementHandle) + .build(); return getFlightInfoForSchema(ticket, descriptor, null); } catch (Exception e) { log.error("getFlightInfoStatement error {}", e); @@ -141,7 +148,9 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS } @Override - public void getStreamPreparedStatement(final CommandPreparedStatementQuery command, final CallContext context, + public void getStreamPreparedStatement( + final CommandPreparedStatementQuery command, + final CallContext context, final ServerStreamListener listener) { log.info("getStreamPreparedStatement {}", command.getPreparedStatementHandle()); executeQuery(command.getPreparedStatementHandle(), listener); @@ -150,84 +159,118 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS private void executeQuery(ByteString hander, final ServerStreamListener listener) { SemanticQueryReq semanticQueryReq = preparedStatementCache.getIfPresent(hander); if (Objects.isNull(semanticQueryReq)) { - listener.error(CallStatus.INTERNAL - .withDescription("Failed to get prepared statement: empty") - .toRuntimeException()); + listener.error( + CallStatus.INTERNAL + .withDescription("Failed to get prepared statement: empty") + .toRuntimeException()); log.error("getStreamPreparedStatement error {}", hander); listener.completed(); return; } - executorService.submit(() -> { - BufferAllocator rootAllocator = new RootAllocator(); - try { - Optional authOpt = semanticQueryReq.getParams().stream() - .filter(p -> p.getName().equals(authenticationConfig.getTokenHttpHeaderKey())).findFirst(); - if (authOpt.isPresent()) { - User user = UserHolder.findUser(authOpt.get().getValue(), - authenticationConfig.getTokenHttpHeaderAppKey()); - SemanticQueryResp resp = queryService.queryByReq(semanticQueryReq, user); - ResultSet resultSet = semanticQueryRespToResultSet(resp, semanticQueryReq.getDataSetId()); - final Schema schema = jdbcToArrowSchema(resultSet.getMetaData(), defaultCalendar); - try (final VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schema, rootAllocator)) { - final VectorLoader loader = new VectorLoader(vectorSchemaRoot); - listener.start(vectorSchemaRoot); - final ArrowVectorIterator iterator = sqlToArrowVectorIterator(resultSet, rootAllocator); - while (iterator.hasNext()) { - final VectorSchemaRoot batch = iterator.next(); - if (batch.getRowCount() == 0) { - break; + executorService.submit( + () -> { + BufferAllocator rootAllocator = new RootAllocator(); + try { + Optional authOpt = + semanticQueryReq.getParams().stream() + .filter( + p -> + p.getName() + .equals( + authenticationConfig + .getTokenHttpHeaderKey())) + .findFirst(); + if (authOpt.isPresent()) { + User user = + UserHolder.findUser( + authOpt.get().getValue(), + authenticationConfig.getTokenHttpHeaderAppKey()); + SemanticQueryResp resp = + queryService.queryByReq(semanticQueryReq, user); + ResultSet resultSet = + semanticQueryRespToResultSet( + resp, semanticQueryReq.getDataSetId()); + final Schema schema = + jdbcToArrowSchema(resultSet.getMetaData(), defaultCalendar); + try (final VectorSchemaRoot vectorSchemaRoot = + VectorSchemaRoot.create(schema, rootAllocator)) { + final VectorLoader loader = new VectorLoader(vectorSchemaRoot); + listener.start(vectorSchemaRoot); + final ArrowVectorIterator iterator = + sqlToArrowVectorIterator(resultSet, rootAllocator); + while (iterator.hasNext()) { + final VectorSchemaRoot batch = iterator.next(); + if (batch.getRowCount() == 0) { + break; + } + final VectorUnloader unloader = new VectorUnloader(batch); + loader.load(unloader.getRecordBatch()); + listener.putNext(); + vectorSchemaRoot.clear(); + } + + listener.putNext(); } - final VectorUnloader unloader = new VectorUnloader(batch); - loader.load(unloader.getRecordBatch()); - listener.putNext(); - vectorSchemaRoot.clear(); } - - listener.putNext(); + } catch (Exception e) { + listener.error( + CallStatus.INTERNAL + .withDescription( + String.format( + "Failed to get exec statement %s", + e.getMessage())) + .toRuntimeException()); + log.error("getStreamPreparedStatement error {}", hander); + } finally { + preparedStatementCache.invalidate(hander); + listener.completed(); + rootAllocator.close(); } - - } - } catch (Exception e) { - listener.error(CallStatus.INTERNAL - .withDescription(String.format("Failed to get exec statement %s", e.getMessage())) - .toRuntimeException()); - log.error("getStreamPreparedStatement error {}", hander); - } finally { - preparedStatementCache.invalidate(hander); - listener.completed(); - rootAllocator.close(); - } - }); + }); } @Override - public void closePreparedStatement(final ActionClosePreparedStatementRequest request, final CallContext context, + public void closePreparedStatement( + final ActionClosePreparedStatementRequest request, + final CallContext context, final StreamListener listener) { log.info("closePreparedStatement {}", request.getPreparedStatementHandle()); listener.onCompleted(); } @Override - public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQuery command, + public FlightInfo getFlightInfoPreparedStatement( + final CommandPreparedStatementQuery command, final CallContext context, final FlightDescriptor descriptor) { return getFlightInfoForSchema(command, descriptor, null); } @Override - public void createPreparedStatement(final ActionCreatePreparedStatementRequest request, final CallContext context, + public void createPreparedStatement( + final ActionCreatePreparedStatementRequest request, + final CallContext context, final StreamListener listener) { prepared(request, context, listener); } private ByteString addPrepared(final CallContext context, String query) throws Exception { if (Arrays.asList(dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey).stream() - .anyMatch(h -> !context.getMiddleware(FlightConstants.HEADER_KEY).headers().containsKey(h))) { - throw new Exception(String.format("Failed to create prepared statement: HeaderCallOption miss %s %s %s", - dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey)); + .anyMatch( + h -> + !context.getMiddleware(FlightConstants.HEADER_KEY) + .headers() + .containsKey(h))) { + throw new Exception( + String.format( + "Failed to create prepared statement: HeaderCallOption miss %s %s %s", + dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey)); } - Long dataSetId = Long.valueOf( - context.getMiddleware(FlightConstants.HEADER_KEY).headers().get(dataSetIdHeaderKey)); + Long dataSetId = + Long.valueOf( + context.getMiddleware(FlightConstants.HEADER_KEY) + .headers() + .get(dataSetIdHeaderKey)); if (StringUtils.isBlank(query)) { throw new Exception("Failed to create prepared statement: query is empty"); } @@ -236,50 +279,60 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS if (StringUtils.isBlank(auth)) { throw new Exception("auth empty"); } - final ByteString preparedStatementHandle = copyFrom( - randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + final ByteString preparedStatementHandle = + copyFrom(randomUUID().toString().getBytes(StandardCharsets.UTF_8)); QuerySqlReq querySqlReq = new QuerySqlReq(); querySqlReq.setDataSetId(dataSetId); querySqlReq.setSql(query); - querySqlReq.setParams(Arrays.asList(new Param(authenticationConfig.getTokenHttpHeaderKey(), auth))); + querySqlReq.setParams( + Arrays.asList(new Param(authenticationConfig.getTokenHttpHeaderKey(), auth))); preparedStatementCache.put(preparedStatementHandle, querySqlReq); - log.info("createPreparedStatement {} {} {} ", preparedStatementHandle, dataSetId, query); + log.info( + "createPreparedStatement {} {} {} ", preparedStatementHandle, dataSetId, query); return preparedStatementHandle; } catch (Exception e) { throw e; } } - private void prepared(final ActionCreatePreparedStatementRequest request, final CallContext context, + private void prepared( + final ActionCreatePreparedStatementRequest request, + final CallContext context, final StreamListener listener) { try { ByteString preparedStatementHandle = addPrepared(context, request.getQuery()); - final ActionCreatePreparedStatementResult result = ActionCreatePreparedStatementResult.newBuilder() - .setDatasetSchema(ByteString.EMPTY) - .setParameterSchema(ByteString.empty()) - .setPreparedStatementHandle(preparedStatementHandle) - .build(); + final ActionCreatePreparedStatementResult result = + ActionCreatePreparedStatementResult.newBuilder() + .setDatasetSchema(ByteString.EMPTY) + .setParameterSchema(ByteString.empty()) + .setPreparedStatementHandle(preparedStatementHandle) + .build(); listener.onNext(new Result(pack(result).toByteArray())); } catch (Exception e) { - listener.onError(CallStatus.INTERNAL - .withDescription(String.format("Failed to create prepared statement: %s", e.getMessage())) - .toRuntimeException()); + listener.onError( + CallStatus.INTERNAL + .withDescription( + String.format( + "Failed to create prepared statement: %s", + e.getMessage())) + .toRuntimeException()); } finally { listener.onCompleted(); } } @Override - protected List determineEndpoints(T t, FlightDescriptor flightDescriptor, - Schema schema) { + protected List determineEndpoints( + T t, FlightDescriptor flightDescriptor, Schema schema) { throw CallStatus.UNIMPLEMENTED.withDescription("Not implemented.").toRuntimeException(); } - private FlightInfo getFlightInfoForSchema(final T request, final FlightDescriptor descriptor, - final Schema schema) { + private FlightInfo getFlightInfoForSchema( + final T request, final FlightDescriptor descriptor, final Schema schema) { final Ticket ticket = new Ticket(pack(request).toByteArray()); Location listenLocation = Location.forGrpcInsecure(host, port); - final List endpoints = singletonList(new FlightEndpoint(ticket, listenLocation)); + final List endpoints = + singletonList(new FlightEndpoint(ticket, listenLocation)); return new FlightInfo(schema, descriptor, endpoints, -1, -1); } @@ -296,7 +349,8 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS return auth; } - private ResultSet semanticQueryRespToResultSet(SemanticQueryResp resp, Long dataSetId) throws SQLException { + private ResultSet semanticQueryRespToResultSet(SemanticQueryResp resp, Long dataSetId) + throws SQLException { RowSetFactory factory = RowSetProvider.newFactory(); CachedRowSet rowset = factory.createCachedRowSet(); RowSetMetaData rowSetMetaData = new RowSetMetaDataImpl(); @@ -305,8 +359,13 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS for (int i = 1; i <= columnNum; i++) { String columnName = resp.getColumns().get(i - 1).getNameEn(); rowSetMetaData.setColumnName(i, columnName); - Optional> valOpt = resp.getResultList().stream() - .filter(r -> r.containsKey(columnName) && Objects.nonNull(r.get(columnName))).findFirst(); + Optional> valOpt = + resp.getResultList().stream() + .filter( + r -> + r.containsKey(columnName) + && Objects.nonNull(r.get(columnName))) + .findFirst(); if (valOpt.isPresent()) { int type = FlightUtils.resolveType(valOpt.get()); rowSetMetaData.setColumnType(i, type); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index f0d5b4aa8..723e7ea1b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -23,6 +23,7 @@ import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MeasureParam; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; @@ -50,11 +51,12 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQuery import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import com.tencent.supersonic.headless.server.pojo.ModelCluster; import com.tencent.supersonic.headless.server.pojo.ModelFilter; +import com.tencent.supersonic.headless.server.service.CollectService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; @@ -63,8 +65,6 @@ import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder; -import com.tencent.supersonic.headless.server.service.CollectService; -import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -110,15 +110,16 @@ public class MetricServiceImpl extends ServiceImpl private ChatLayerService chatLayerService; - public MetricServiceImpl(MetricRepository metricRepository, - ModelService modelService, - AliasGenerateHelper aliasGenerateHelper, - CollectService collectService, - DataSetService dataSetService, - ApplicationEventPublisher eventPublisher, - DimensionService dimensionService, - TagMetaService tagMetaService, - @Lazy ChatLayerService chatLayerService) { + public MetricServiceImpl( + MetricRepository metricRepository, + ModelService modelService, + AliasGenerateHelper aliasGenerateHelper, + CollectService collectService, + DataSetService dataSetService, + ApplicationEventPublisher eventPublisher, + DimensionService dimensionService, + TagMetaService tagMetaService, + @Lazy ChatLayerService chatLayerService) { this.metricRepository = metricRepository; this.modelService = modelService; this.aliasGenerateHelper = aliasGenerateHelper; @@ -148,18 +149,27 @@ public class MetricServiceImpl extends ServiceImpl } Long modelId = metricReqs.get(0).getModelId(); List metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId))); - Map bizNameMap = metricResps.stream() - .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); - Map nameMap = metricResps.stream() - .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); - List metricToInsert = metricReqs.stream() - .filter(metric -> !bizNameMap.containsKey(metric.getBizName()) - && !nameMap.containsKey(metric.getName())).collect(Collectors.toList()); + Map bizNameMap = + metricResps.stream() + .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); + Map nameMap = + metricResps.stream() + .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); + List metricToInsert = + metricReqs.stream() + .filter( + metric -> + !bizNameMap.containsKey(metric.getBizName()) + && !nameMap.containsKey(metric.getName())) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(metricToInsert)) { return; } - List metricDOS = metricToInsert.stream().peek(metric -> metric.createdBy(user.getName())) - .map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); + List metricDOS = + metricToInsert.stream() + .peek(metric -> metric.createdBy(user.getName())) + .map(MetricConverter::convert2MetricDO) + .collect(Collectors.toList()); metricRepository.createMetricBatch(metricDOS); sendEventBatch(metricDOS, EventType.ADD); } @@ -191,13 +201,15 @@ public class MetricServiceImpl extends ServiceImpl if (CollectionUtils.isEmpty(metricDOS)) { return; } - metricDOS = metricDOS.stream() - .peek(metricDO -> { - metricDO.setStatus(metaBatchReq.getStatus()); - metricDO.setUpdatedAt(new Date()); - metricDO.setUpdatedBy(user.getName()); - }) - .collect(Collectors.toList()); + metricDOS = + metricDOS.stream() + .peek( + metricDO -> { + metricDO.setStatus(metaBatchReq.getStatus()); + metricDO.setUpdatedAt(new Date()); + metricDO.setUpdatedBy(user.getName()); + }) + .collect(Collectors.toList()); metricRepository.batchUpdateStatus(metricDOS); if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus()) || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) { @@ -286,12 +298,12 @@ public class MetricServiceImpl extends ServiceImpl @Override public PageInfo queryMetricMarket(PageMetricReq pageMetricReq, User user) { - //search by whole text + // search by whole text PageInfo metricRespPageInfo = queryMetric(pageMetricReq, user); if (metricRespPageInfo.hasContent() || StringUtils.isBlank(pageMetricReq.getKey())) { return metricRespPageInfo; } - //search by text split + // search by text split QueryMapReq queryMapReq = new QueryMapReq(); queryMapReq.setQueryText(pageMetricReq.getKey()); queryMapReq.setUser(user); @@ -301,14 +313,21 @@ public class MetricServiceImpl extends ServiceImpl if (CollectionUtils.isEmpty(dataSetMapInfoMap)) { return metricRespPageInfo; } - Map result = dataSetMapInfoMap.values().stream() - .map(DataSetMapInfo::getMapFields) - .filter(Objects::nonNull) - .flatMap(Collection::stream).filter(schemaElementMatch -> - SchemaElementType.METRIC.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toMap(schemaElementMatch -> - schemaElementMatch.getElement().getId(), SchemaElementMatch::getSimilarity, - (existingValue, newValue) -> existingValue)); + Map result = + dataSetMapInfoMap.values().stream() + .map(DataSetMapInfo::getMapFields) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .filter( + schemaElementMatch -> + SchemaElementType.METRIC.equals( + schemaElementMatch.getElement().getType())) + .collect( + Collectors.toMap( + schemaElementMatch -> + schemaElementMatch.getElement().getId(), + SchemaElementMatch::getSimilarity, + (existingValue, newValue) -> existingValue)); List metricIds = new ArrayList<>(result.keySet()); if (CollectionUtils.isEmpty(result.keySet())) { return metricRespPageInfo; @@ -329,17 +348,19 @@ public class MetricServiceImpl extends ServiceImpl metricFilter.setUserName(user.getName()); BeanUtils.copyProperties(pageMetricReq, metricFilter); if (!CollectionUtils.isEmpty(pageMetricReq.getDomainIds())) { - List modelResps = modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds()); - List modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); + List modelResps = + modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds()); + List modelIds = + modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); pageMetricReq.getModelIds().addAll(modelIds); } metricFilter.setModelIds(pageMetricReq.getModelIds()); List collectIds = getCollectIds(pageMetricReq, user); List idsToFilter = getIdsToFilter(pageMetricReq, collectIds); metricFilter.setIds(idsToFilter); - PageInfo metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(), - pageMetricReq.getPageSize()) - .doSelectPageInfo(() -> queryMetric(metricFilter)); + PageInfo metricDOPageInfo = + PageHelper.startPage(pageMetricReq.getCurrent(), pageMetricReq.getPageSize()) + .doSelectPageInfo(() -> queryMetric(metricFilter)); PageInfo pageInfo = new PageInfo<>(); BeanUtils.copyProperties(metricDOPageInfo, pageInfo); List metricResps = convertList(metricDOPageInfo.getList(), collectIds); @@ -378,8 +399,10 @@ public class MetricServiceImpl extends ServiceImpl } private List getCollectIds(PageMetricReq pageMetricReq, User user) { - List collectList = collectService.getCollectionList(user.getName(), TypeEnums.METRIC); - List collectIds = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList()); + List collectList = + collectService.getCollectionList(user.getName(), TypeEnums.METRIC); + List collectIds = + collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList()); if (pageMetricReq.isHasCollect()) { if (CollectionUtils.isEmpty(collectIds)) { return Lists.newArrayList(-1L); @@ -411,14 +434,20 @@ public class MetricServiceImpl extends ServiceImpl return new ArrayList<>(metricRespFiltered); } - private boolean filterByField(List metricResps, MetricResp metricResp, - List fields, Set metricRespFiltered) { + private boolean filterByField( + List metricResps, + MetricResp metricResp, + List fields, + Set metricRespFiltered) { if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { - List ids = metricResp.getMetricDefineByMetricParams().getMetrics() - .stream().map(MetricParam::getId).collect(Collectors.toList()); - List metricById = metricResps.stream() - .filter(metric -> ids.contains(metric.getId())) - .collect(Collectors.toList()); + List ids = + metricResp.getMetricDefineByMetricParams().getMetrics().stream() + .map(MetricParam::getId) + .collect(Collectors.toList()); + List metricById = + metricResps.stream() + .filter(metric -> ids.contains(metric.getId())) + .collect(Collectors.toList()); for (MetricResp metric : metricById) { if (filterByField(metricResps, metric, fields, metricRespFiltered)) { metricRespFiltered.add(metricResp); @@ -432,10 +461,12 @@ public class MetricServiceImpl extends ServiceImpl } } else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) { List measures = metricResp.getMetricDefineByMeasureParams().getMeasures(); - List fieldNameDepended = measures.stream().map(MeasureParam::getBizName) - //measure bizName = model bizName_fieldName - .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", "")) - .collect(Collectors.toList()); + List fieldNameDepended = + measures.stream() + .map(MeasureParam::getBizName) + // measure bizName = model bizName_fieldName + .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", "")) + .collect(Collectors.toList()); if (fields.stream().anyMatch(fieldNameDepended::contains)) { metricRespFiltered.add(metricResp); return true; @@ -449,9 +480,12 @@ public class MetricServiceImpl extends ServiceImpl MetricFilter metricFilter = new MetricFilter(); metricFilter.setModelIds(Lists.newArrayList(modelId)); List metricResps = getMetrics(metricFilter); - return metricResps.stream().filter(metricResp -> - MetricDefineType.FIELD.equals(metricResp.getMetricDefineType()) - || MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) + return metricResps.stream() + .filter( + metricResp -> + MetricDefineType.FIELD.equals(metricResp.getMetricDefineType()) + || MetricDefineType.MEASURE.equals( + metricResp.getMetricDefineType())) .collect(Collectors.toList()); } @@ -460,7 +494,8 @@ public class MetricServiceImpl extends ServiceImpl if (CollectionUtils.isEmpty(modelResps)) { return; } - Set modelIdSet = modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet()); + Set modelIdSet = + modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet()); for (MetricResp metricResp : metricResps) { if (modelIdSet.contains(metricResp.getModelId())) { metricResp.setHasAdminRes(true); @@ -488,11 +523,12 @@ public class MetricServiceImpl extends ServiceImpl if (metricDO == null) { return null; } - ModelFilter modelFilter = new ModelFilter(true, - Lists.newArrayList(metricDO.getModelId())); + ModelFilter modelFilter = new ModelFilter(true, Lists.newArrayList(metricDO.getModelId())); Map modelMap = modelService.getModelMap(modelFilter); - List collectList = collectService.getCollectionList(user.getName(), TypeEnums.METRIC); - List collect = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList()); + List collectList = + collectService.getCollectionList(user.getName(), TypeEnums.METRIC); + List collect = + collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList()); MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, modelMap, collect); fillAdminRes(Lists.newArrayList(metricResp), user); return metricResp; @@ -510,11 +546,20 @@ public class MetricServiceImpl extends ServiceImpl @Override public List mockAlias(MetricBaseReq metricReq, String mockType, User user) { - String mockAlias = aliasGenerateHelper.generateAlias(mockType, metricReq.getName(), metricReq.getBizName(), "", - metricReq.getDescription()); - String ret = mockAlias.replaceAll("`", "").replace("json", "").replace("\n", "").replace(" ", ""); - return JSONObject.parseObject(ret, new TypeReference>() { - }); + String mockAlias = + aliasGenerateHelper.generateAlias( + mockType, + metricReq.getName(), + metricReq.getBizName(), + "", + metricReq.getDescription()); + String ret = + mockAlias + .replaceAll("`", "") + .replace("json", "") + .replace("\n", "") + .replace(" ", ""); + return JSONObject.parseObject(ret, new TypeReference>() {}); } @Override @@ -523,8 +568,9 @@ public class MetricServiceImpl extends ServiceImpl if (CollectionUtils.isEmpty(metricResps)) { return new HashSet<>(); } - return metricResps.stream().flatMap(metricResp -> - metricResp.getClassifications().stream()).collect(Collectors.toSet()); + return metricResps.stream() + .flatMap(metricResp -> metricResp.getClassifications().stream()) + .collect(Collectors.toSet()); } @Override @@ -535,9 +581,12 @@ public class MetricServiceImpl extends ServiceImpl return drillDownDimensions; } if (metricResp.getRelateDimension() != null - && !CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { - for (DrillDownDimension drillDownDimension : metricResp.getRelateDimension().getDrillDownDimensions()) { - if (drillDownDimension.isInheritedFromModel() && !drillDownDimension.isNecessary()) { + && !CollectionUtils.isEmpty( + metricResp.getRelateDimension().getDrillDownDimensions())) { + for (DrillDownDimension drillDownDimension : + metricResp.getRelateDimension().getDrillDownDimensions()) { + if (drillDownDimension.isInheritedFromModel() + && !drillDownDimension.isNecessary()) { continue; } drillDownDimensions.add(drillDownDimension); @@ -548,8 +597,10 @@ public class MetricServiceImpl extends ServiceImpl return drillDownDimensions; } for (DrillDownDimension drillDownDimension : modelResp.getDrillDownDimensions()) { - if (!drillDownDimensions.stream().map(DrillDownDimension::getDimensionId) - .collect(Collectors.toList()).contains(drillDownDimension.getDimensionId())) { + if (!drillDownDimensions.stream() + .map(DrillDownDimension::getDimensionId) + .collect(Collectors.toList()) + .contains(drillDownDimension.getDimensionId())) { drillDownDimension.setInheritedFromModel(true); drillDownDimensions.add(drillDownDimension); } @@ -588,23 +639,29 @@ public class MetricServiceImpl extends ServiceImpl MetaFilter metaFilter = new MetaFilter(); metaFilter.setModelIds(Lists.newArrayList(modelId)); List metricResps = getMetrics(metaFilter); - Map bizNameMap = metricResps.stream() - .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); - Map nameMap = metricResps.stream() - .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); + Map bizNameMap = + metricResps.stream() + .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); + Map nameMap = + metricResps.stream() + .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); for (MetricBaseReq metricReq : metricReqs) { if (bizNameMap.containsKey(metricReq.getBizName())) { MetricResp metricResp = bizNameMap.get(metricReq.getBizName()); if (!metricResp.getId().equals(metricReq.getId())) { - throw new RuntimeException(String.format("该模型下存在相同的指标字段名:%s 创建人:%s", - metricReq.getBizName(), metricResp.getCreatedBy())); + throw new RuntimeException( + String.format( + "该模型下存在相同的指标字段名:%s 创建人:%s", + metricReq.getBizName(), metricResp.getCreatedBy())); } } if (nameMap.containsKey(metricReq.getName())) { MetricResp metricResp = nameMap.get(metricReq.getName()); if (!metricResp.getId().equals(metricReq.getId())) { - throw new RuntimeException(String.format("该模型下存在相同的指标名:%s 创建人:%s", - metricReq.getName(), metricResp.getCreatedBy())); + throw new RuntimeException( + String.format( + "该模型下存在相同的指标名:%s 创建人:%s", + metricReq.getName(), metricResp.getCreatedBy())); } } } @@ -616,14 +673,18 @@ public class MetricServiceImpl extends ServiceImpl private List convertList(List metricDOS, List collect) { List metricResps = Lists.newArrayList(); - List modelIds = metricDOS.stream().map(MetricDO::getModelId) - .collect(Collectors.toList()); + List modelIds = + metricDOS.stream().map(MetricDO::getModelId).collect(Collectors.toList()); ModelFilter modelFilter = new ModelFilter(false, modelIds); Map modelMap = modelService.getModelMap(modelFilter); if (!CollectionUtils.isEmpty(metricDOS)) { - metricResps = metricDOS.stream() - .map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect)) - .collect(Collectors.toList()); + metricResps = + metricDOS.stream() + .map( + metricDO -> + MetricConverter.convert2MetricResp( + metricDO, modelMap, collect)) + .collect(Collectors.toList()); } return metricResps; } @@ -650,8 +711,8 @@ public class MetricServiceImpl extends ServiceImpl } private DataEvent getDataEvent(List metricDOS, EventType eventType) { - List dataItems = metricDOS.stream().map(this::getDataItem) - .collect(Collectors.toList()); + List dataItems = + metricDOS.stream().map(this::getDataItem).collect(Collectors.toList()); return new DataEvent(this, dataItems, eventType); } @@ -661,23 +722,28 @@ public class MetricServiceImpl extends ServiceImpl } private void sendEvent(DataItem dataItem, EventType eventType) { - eventPublisher.publishEvent(new DataEvent(this, - Lists.newArrayList(dataItem), eventType)); + eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType)); } private DataItem getDataItem(MetricDO metricDO) { - MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, - new HashMap<>(), Lists.newArrayList()); + MetricResp metricResp = + MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList()); fillDefaultAgg(metricResp); - return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE).name(metricDO.getName()) + return DataItem.builder() + .id(metricDO.getId() + Constants.UNDERLINE) + .name(metricDO.getName()) .bizName(metricDO.getBizName()) .modelId(metricDO.getModelId() + Constants.UNDERLINE) - .type(TypeEnums.METRIC).defaultAgg(metricResp.getDefaultAgg()).build(); + .type(TypeEnums.METRIC) + .defaultAgg(metricResp.getDefaultAgg()) + .build(); } @Override - public void batchFillMetricDefaultAgg(List metricResps, List modelResps) { - Map modelRespMap = modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m)); + public void batchFillMetricDefaultAgg( + List metricResps, List modelResps) { + Map modelRespMap = + modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m)); for (MetricResp metricResp : metricResps) { fillDefaultAgg(metricResp, modelRespMap.get(metricResp.getModelId())); } @@ -696,8 +762,9 @@ public class MetricServiceImpl extends ServiceImpl } private String getDefaultAgg(MetricResp metricResp, ModelResp modelResp) { - if (modelResp == null || (Objects.nonNull(metricResp.getDefaultAgg()) && !metricResp.getDefaultAgg() - .isEmpty())) { + if (modelResp == null + || (Objects.nonNull(metricResp.getDefaultAgg()) + && !metricResp.getDefaultAgg().isEmpty())) { return metricResp.getDefaultAgg(); } // FIELD define will get from expr @@ -706,10 +773,11 @@ public class MetricServiceImpl extends ServiceImpl } // METRIC define will get from first metric if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { - if (!CollectionUtils.isEmpty( - metricResp.getMetricDefineByMetricParams().getMetrics())) { - MetricParam metricParam = metricResp.getMetricDefineByMetricParams().getMetrics().get(0); - MetricResp firstMetricResp = getMetric(modelResp.getDomainId(), metricParam.getBizName()); + if (!CollectionUtils.isEmpty(metricResp.getMetricDefineByMetricParams().getMetrics())) { + MetricParam metricParam = + metricResp.getMetricDefineByMetricParams().getMetrics().get(0); + MetricResp firstMetricResp = + getMetric(modelResp.getDomainId(), metricParam.getBizName()); if (Objects.nonNull(firstMetricResp)) { return getDefaultAgg(firstMetricResp, modelResp); } @@ -718,8 +786,8 @@ public class MetricServiceImpl extends ServiceImpl } // Measure define will get from first measure List measures = modelResp.getModelDetail().getMeasures(); - List measureParams = metricResp.getMetricDefineByMeasureParams() - .getMeasures(); + List measureParams = + metricResp.getMetricDefineByMeasureParams().getMeasures(); if (CollectionUtils.isEmpty(measureParams)) { return ""; } @@ -735,32 +803,41 @@ public class MetricServiceImpl extends ServiceImpl @Override public QueryStructReq convert(QueryMetricReq queryMetricReq) { - //1. If a domainId exists, the modelIds obtained from the domainId. + // 1. If a domainId exists, the modelIds obtained from the domainId. Set modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq); - //2. get metrics and dimensions + // 2. get metrics and dimensions List metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId); List dimensionResps = getDimensionResps(modelIdsByDomainId); - Map dimensionRespMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getId, d -> d)); + Map dimensionRespMap = + dimensionResps.stream().collect(Collectors.toMap(DimensionResp::getId, d -> d)); - //3. choose ModelCluster + // 3. choose ModelCluster Set modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps); ModelCluster modelCluster = getModelCluster(metricResps, modelIds); if (modelCluster == null) { - throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics"); + throw new IllegalArgumentException( + "Invalid input parameters, unable to obtain valid metrics"); } if (!modelCluster.isContainsPartitionDimensions()) { queryMetricReq.setDateInfo(null); } - //4. set groups - List dimensionBizNames = dimensionResps.stream() - .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) - .filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName()) - || queryMetricReq.getDimensionNames().contains(entry.getBizName()) - || queryMetricReq.getDimensionIds().contains(entry.getId())) - .map(SchemaItem::getBizName).collect(Collectors.toList()); + // 4. set groups + List dimensionBizNames = + dimensionResps.stream() + .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) + .filter( + entry -> + queryMetricReq.getDimensionNames().contains(entry.getName()) + || queryMetricReq + .getDimensionNames() + .contains(entry.getBizName()) + || queryMetricReq + .getDimensionIds() + .contains(entry.getId())) + .map(SchemaItem::getBizName) + .collect(Collectors.toList()); QueryStructReq queryStructReq = new QueryStructReq(); DateConf dateInfo = queryMetricReq.getDateInfo(); @@ -770,12 +847,15 @@ public class MetricServiceImpl extends ServiceImpl if (!CollectionUtils.isEmpty(dimensionBizNames)) { queryStructReq.getGroups().addAll(dimensionBizNames); } - //5. set aggregators - List metricBizNames = metricResps.stream() - .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) - .map(SchemaItem::getBizName).collect(Collectors.toList()); + // 5. set aggregators + List metricBizNames = + metricResps.stream() + .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) + .map(SchemaItem::getBizName) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(metricBizNames)) { - throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics"); + throw new IllegalArgumentException( + "Invalid input parameters, unable to obtain valid metrics"); } List aggregators = new ArrayList<>(); for (String metricBizName : metricBizNames) { @@ -785,7 +865,7 @@ public class MetricServiceImpl extends ServiceImpl } queryStructReq.setAggregators(aggregators); queryStructReq.setLimit(queryMetricReq.getLimit()); - //6. set modelIds + // 6. set modelIds for (Long modelId : modelCluster.getModelIds()) { queryStructReq.addModelId(modelId); } @@ -799,44 +879,51 @@ public class MetricServiceImpl extends ServiceImpl } } queryStructReq.setDimensionFilters(filters); - //7. set dateInfo + // 7. set dateInfo queryStructReq.setDateInfo(dateInfo); return queryStructReq; } private ModelCluster getModelCluster(List metricResps, Set modelIds) { - Map modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds)); + Map modelClusterMap = + ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds)); Map> modelClusterToMatchCount = new HashMap<>(); for (ModelCluster modelCluster : modelClusterMap.values()) { for (MetricResp metricResp : metricResps) { if (modelCluster.getModelIds().contains(metricResp.getModelId())) { - modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>()) + modelClusterToMatchCount + .computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>()) .add(metricResp); } } } - String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream() - .max(Comparator.comparingInt(entry -> entry.getValue().size())) - .map(Map.Entry::getKey) - .orElse(null); + String keyWithMaxSize = + modelClusterToMatchCount.entrySet().stream() + .max(Comparator.comparingInt(entry -> entry.getValue().size())) + .map(Map.Entry::getKey) + .orElse(null); return modelClusterMap.get(keyWithMaxSize); } - private Set getModelIds(Set modelIdsByDomainId, List metricResps, - List dimensionResps) { + private Set getModelIds( + Set modelIdsByDomainId, + List metricResps, + List dimensionResps) { Set result = new HashSet<>(); if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) { result.addAll(modelIdsByDomainId); return result; } - Set metricModelIds = metricResps.stream().map(entry -> entry.getModelId()) - .collect(Collectors.toSet()); + Set metricModelIds = + metricResps.stream().map(entry -> entry.getModelId()).collect(Collectors.toSet()); result.addAll(metricModelIds); - Set dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId()) - .collect(Collectors.toSet()); + Set dimensionModelIds = + dimensionResps.stream() + .map(entry -> entry.getModelId()) + .collect(Collectors.toSet()); result.addAll(dimensionModelIds); return result; } @@ -855,9 +942,9 @@ public class MetricServiceImpl extends ServiceImpl } private Set getModelIdsByDomainId(QueryMetricReq queryMetricReq) { - List modelResps = modelService.getAllModelByDomainIds( - Collections.singletonList(queryMetricReq.getDomainId())); + List modelResps = + modelService.getAllModelByDomainIds( + Collections.singletonList(queryMetricReq.getDomainId())); return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelRelaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelRelaServiceImpl.java index 0df24e10d..5b7ee2660 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelRelaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelRelaServiceImpl.java @@ -18,8 +18,8 @@ import java.util.List; import java.util.stream.Collectors; @Service -public class ModelRelaServiceImpl - extends ServiceImpl implements ModelRelaService { +public class ModelRelaServiceImpl extends ServiceImpl + implements ModelRelaService { @Override public void save(ModelRela modelRela, User user) { @@ -48,7 +48,9 @@ public class ModelRelaServiceImpl if (CollectionUtils.isEmpty(modelIds)) { return Lists.newArrayList(); } - wrapper.lambda().in(ModelRelaDO::getFromModelId, modelIds).or() + wrapper.lambda() + .in(ModelRelaDO::getFromModelId, modelIds) + .or() .in(ModelRelaDO::getToModelId, modelIds); return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); } @@ -61,7 +63,8 @@ public class ModelRelaServiceImpl private ModelRela convert(ModelRelaDO modelRelaDO) { ModelRela modelRela = new ModelRela(); BeanMapper.mapper(modelRelaDO, modelRela); - modelRela.setJoinConditions(JSONObject.parseArray(modelRelaDO.getJoinCondition(), JoinCondition.class)); + modelRela.setJoinConditions( + JSONObject.parseArray(modelRelaDO.getJoinCondition(), JoinCondition.class)); return modelRela; } @@ -71,5 +74,4 @@ public class ModelRelaServiceImpl modelRelaDO.setJoinCondition(JSONObject.toJSONString(modelRelaReq.getJoinConditions())); return modelRelaDO; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index c6b3bcfc1..d7326a119 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -13,31 +13,31 @@ import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.Measure; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.request.DateInfoReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; import com.tencent.supersonic.headless.server.persistence.repository.ModelRepository; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; +import com.tencent.supersonic.headless.server.service.DataSetService; +import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.DatabaseService; -import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.utils.ModelConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; @@ -58,7 +58,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - @Service @Slf4j public class ModelServiceImpl implements ModelService { @@ -79,14 +78,15 @@ public class ModelServiceImpl implements ModelService { private DateInfoRepository dateInfoRepository; - public ModelServiceImpl(ModelRepository modelRepository, - DatabaseService databaseService, - @Lazy DimensionService dimensionService, - @Lazy MetricService metricService, - DomainService domainService, - UserService userService, - DataSetService dataSetService, - DateInfoRepository dateInfoRepository) { + public ModelServiceImpl( + ModelRepository modelRepository, + DatabaseService databaseService, + @Lazy DimensionService dimensionService, + @Lazy MetricService metricService, + DomainService domainService, + UserService userService, + DataSetService dataSetService, + DateInfoRepository dateInfoRepository) { this.modelRepository = modelRepository; this.databaseService = databaseService; this.dimensionService = dimensionService; @@ -124,10 +124,12 @@ public class ModelServiceImpl implements ModelService { public List getModelList(MetaFilter metaFilter) { ModelFilter modelFilter = new ModelFilter(); BeanUtils.copyProperties(metaFilter, modelFilter); - List modelResps = ModelConverter.convertList(modelRepository.getModelList(modelFilter)); + List modelResps = + ModelConverter.convertList(modelRepository.getModelList(modelFilter)); if (modelFilter.getDataSetId() != null) { DataSetResp dataSetResp = dataSetService.getDataSet(modelFilter.getDataSetId()); - return modelResps.stream().filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId())) + return modelResps.stream() + .filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId())) .collect(Collectors.toList()); } return modelResps; @@ -140,7 +142,8 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(modelResps)) { return map; } - return modelResps.stream().collect(Collectors.toMap(ModelResp::getId, a -> a, (k1, k2) -> k1)); + return modelResps.stream() + .collect(Collectors.toMap(ModelResp::getId, a -> a, (k1, k2) -> k1)); } @Override @@ -182,8 +185,10 @@ public class ModelServiceImpl implements ModelService { metaFilter.setFieldsDepend(fieldRemovedReq.getFields()); List metricResps = metricService.getMetrics(metaFilter); List dimensionResps = dimensionService.getDimensions(metaFilter); - return UnAvailableItemResp.builder().dimensionResps(dimensionResps) - .metricResps(metricResps).build(); + return UnAvailableItemResp.builder() + .dimensionResps(dimensionResps) + .metricResps(metricResps) + .build(); } private void batchCreateDimension(ModelDO modelDO, User user) throws Exception { @@ -199,7 +204,9 @@ public class ModelServiceImpl implements ModelService { private void checkParams(ModelReq modelReq) { String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(modelReq.getName()); if (StringUtils.isNotBlank(forbiddenCharacters)) { - String message = String.format("模型名称[%s]包含特殊字符(%s), 请修改", modelReq.getName(), forbiddenCharacters); + String message = + String.format( + "模型名称[%s]包含特殊字符(%s), 请修改", modelReq.getName(), forbiddenCharacters); throw new InvalidArgumentException(message); } @@ -215,10 +222,14 @@ public class ModelServiceImpl implements ModelService { throw new InvalidArgumentException("缺少维度信息"); } for (Measure measure : measures) { - String measureForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(measure.getName()); + String measureForbiddenCharacters = + NameCheckUtils.findForbiddenCharacters(measure.getName()); if (StringUtils.isNotBlank(measure.getName()) && StringUtils.isNotBlank(measureForbiddenCharacters)) { - String message = String.format("度量[%s]包含特殊字符(%s), 请修改", measure.getName(), measureForbiddenCharacters); + String message = + String.format( + "度量[%s]包含特殊字符(%s), 请修改", + measure.getName(), measureForbiddenCharacters); throw new InvalidArgumentException(message); } } @@ -226,16 +237,21 @@ public class ModelServiceImpl implements ModelService { String dimForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dim.getName()); if (StringUtils.isNotBlank(dim.getName()) && StringUtils.isNotBlank(dimForbiddenCharacters)) { - String message = String.format("维度[%s]包含特殊字符(%s), 请修改", dim.getName(), dimForbiddenCharacters); + String message = + String.format( + "维度[%s]包含特殊字符(%s), 请修改", dim.getName(), dimForbiddenCharacters); throw new InvalidArgumentException(message); } } for (Identify identify : identifies) { - String identifyForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(identify.getName()); + String identifyForbiddenCharacters = + NameCheckUtils.findForbiddenCharacters(identify.getName()); if (StringUtils.isNotBlank(identify.getName()) && StringUtils.isNotBlank(identifyForbiddenCharacters)) { - String message = String.format("主键/外键[%s]包含特殊字符(%s), 请修改", identify.getName(), - identifyForbiddenCharacters); + String message = + String.format( + "主键/外键[%s]包含特殊字符(%s), 请修改", + identify.getName(), identifyForbiddenCharacters); throw new InvalidArgumentException(message); } } @@ -283,22 +299,29 @@ public class ModelServiceImpl implements ModelService { public List getModelListWithAuth(User user, Long domainId, AuthType authType) { List modelResps = getModelAuthList(user, domainId, authType); Set modelRespSet = new HashSet<>(modelResps); - List modelRespsAuthInheritDomain = getModelRespAuthInheritDomain(user, domainId, authType); + List modelRespsAuthInheritDomain = + getModelRespAuthInheritDomain(user, domainId, authType); modelRespSet.addAll(modelRespsAuthInheritDomain); - return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId)) + return modelRespSet.stream() + .sorted(Comparator.comparingLong(ModelResp::getId)) .collect(Collectors.toList()); } - public List getModelRespAuthInheritDomain(User user, Long domainId, AuthType authType) { - List domainIds = domainService.getDomainAuthSet(user, authType) - .stream().filter(domainResp -> { - if (domainId == null) { - return true; - } else { - return domainId.equals(domainResp.getId()) || domainId.equals(domainResp.getParentId()); - } - }).map(DomainResp::getId) - .collect(Collectors.toList()); + public List getModelRespAuthInheritDomain( + User user, Long domainId, AuthType authType) { + List domainIds = + domainService.getDomainAuthSet(user, authType).stream() + .filter( + domainResp -> { + if (domainId == null) { + return true; + } else { + return domainId.equals(domainResp.getId()) + || domainId.equals(domainResp.getParentId()); + } + }) + .map(DomainResp::getId) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(domainIds)) { return Lists.newArrayList(); } @@ -317,14 +340,16 @@ public class ModelServiceImpl implements ModelService { Set orgIds = userService.getUserAllOrgId(user.getName()); List modelWithAuth = Lists.newArrayList(); if (authTypeEnum.equals(AuthType.ADMIN)) { - modelWithAuth = modelResps.stream() - .filter(modelResp -> checkAdminPermission(orgIds, user, modelResp)) - .collect(Collectors.toList()); + modelWithAuth = + modelResps.stream() + .filter(modelResp -> checkAdminPermission(orgIds, user, modelResp)) + .collect(Collectors.toList()); } if (authTypeEnum.equals(AuthType.VISIBLE)) { - modelWithAuth = modelResps.stream() - .filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp)) - .collect(Collectors.toList()); + modelWithAuth = + modelResps.stream() + .filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp)) + .collect(Collectors.toList()); } return modelWithAuth; } @@ -341,14 +366,16 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(modelResps)) { return modelResps; } - return modelResps.stream().filter(modelResp -> - domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList()); + return modelResps.stream() + .filter(modelResp -> domainIds.contains(modelResp.getDomainId())) + .collect(Collectors.toList()); } @Override public List getAllModelByDomainIds(List domainIds) { Set domainResps = domainService.getDomainChildren(domainIds); - List allDomainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); + List allDomainIds = + domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); return getModelByDomainIds(allDomainIds); } @@ -405,21 +432,35 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(modelDOS)) { return; } - modelDOS = modelDOS.stream() - .peek(modelDO -> { - modelDO.setStatus(metaBatchReq.getStatus()); - modelDO.setUpdatedAt(new Date()); - modelDO.setUpdatedBy(user.getName()); - if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus()) - || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) { - metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()), EventType.DELETE); - dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()), EventType.DELETE); - } else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) { - metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()), EventType.ADD); - dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()), EventType.ADD); - } - }) - .collect(Collectors.toList()); + modelDOS = + modelDOS.stream() + .peek( + modelDO -> { + modelDO.setStatus(metaBatchReq.getStatus()); + modelDO.setUpdatedAt(new Date()); + modelDO.setUpdatedBy(user.getName()); + if (StatusEnum.OFFLINE + .getCode() + .equals(metaBatchReq.getStatus()) + || StatusEnum.DELETED + .getCode() + .equals(metaBatchReq.getStatus())) { + metricService.sendMetricEventBatch( + Lists.newArrayList(modelDO.getId()), + EventType.DELETE); + dimensionService.sendDimensionEventBatch( + Lists.newArrayList(modelDO.getId()), + EventType.DELETE); + } else if (StatusEnum.ONLINE + .getCode() + .equals(metaBatchReq.getStatus())) { + metricService.sendMetricEventBatch( + Lists.newArrayList(modelDO.getId()), EventType.ADD); + dimensionService.sendDimensionEventBatch( + Lists.newArrayList(modelDO.getId()), EventType.ADD); + } + }) + .collect(Collectors.toList()); modelRepository.batchUpdate(modelDOS); } @@ -429,12 +470,14 @@ public class ModelServiceImpl implements ModelService { private List convert(List dateInfoDOList) { List dateInfoCommendList = new ArrayList<>(); - dateInfoDOList.forEach(dateInfoDO -> { - DateInfoReq dateInfoCommend = new DateInfoReq(); - BeanUtils.copyProperties(dateInfoDO, dateInfoCommend); - dateInfoCommend.setUnavailableDateList(JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class)); - dateInfoCommendList.add(dateInfoCommend); - }); + dateInfoDOList.forEach( + dateInfoDO -> { + DateInfoReq dateInfoCommend = new DateInfoReq(); + BeanUtils.copyProperties(dateInfoDO, dateInfoCommend); + dateInfoCommend.setUnavailableDateList( + JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class)); + dateInfoCommendList.add(dateInfoCommend); + }); return dateInfoCommendList; } @@ -459,7 +502,8 @@ public class ModelServiceImpl implements ModelService { return false; } - public static boolean checkDataSetPermission(Set orgIds, User user, ModelResp modelResp) { + public static boolean checkDataSetPermission( + Set orgIds, User user, ModelResp modelResp) { if (checkAdminPermission(orgIds, user, modelResp)) { return true; } @@ -485,5 +529,4 @@ public class ModelServiceImpl implements ModelService { } return false; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryRuleServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryRuleServiceImpl.java index c6a633980..f12ebf1d6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryRuleServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryRuleServiceImpl.java @@ -26,7 +26,8 @@ public class QueryRuleServiceImpl implements QueryRuleService { private final QueryRuleRepository queryRuleRepository; private final DataSetService dataSetService; - public QueryRuleServiceImpl(QueryRuleRepository queryRuleRepository, DataSetService dataSetService) { + public QueryRuleServiceImpl( + QueryRuleRepository queryRuleRepository, DataSetService dataSetService) { this.queryRuleRepository = queryRuleRepository; this.dataSetService = dataSetService; } @@ -87,7 +88,8 @@ public class QueryRuleServiceImpl implements QueryRuleService { String userName = user.getName(); if (Objects.nonNull(queryRuleReq.getDataSetId())) { DataSetResp dataSet = dataSetService.getDataSet(queryRuleReq.getDataSetId()); - if (dataSet.getAdmins().contains(userName) || dataSet.getCreatedBy().equalsIgnoreCase(userName)) { + if (dataSet.getAdmins().contains(userName) + || dataSet.getCreatedBy().equalsIgnoreCase(userName)) { log.debug(String.format("user:%s, queryRuleReq:%s", userName, queryRuleReq)); return; } @@ -99,12 +101,12 @@ public class QueryRuleServiceImpl implements QueryRuleService { String userName = user.getName(); if (Objects.nonNull(queryRuleDO.getDataSetId())) { DataSetResp dataSet = dataSetService.getDataSet(queryRuleDO.getDataSetId()); - if (dataSet.getAdmins().contains(userName) || dataSet.getCreatedBy().equalsIgnoreCase(userName)) { + if (dataSet.getAdmins().contains(userName) + || dataSet.getCreatedBy().equalsIgnoreCase(userName)) { log.debug(String.format("user:%s, queryRuleDO:%s", userName, queryRuleDO)); return; } throw new RuntimeException("用户暂无权限变更数据集的规则, 请确认"); } } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java index d240cbf92..da318eeea 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java @@ -21,8 +21,8 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; import com.tencent.supersonic.headless.chat.mapper.MatchText; import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType; import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy; -import com.tencent.supersonic.headless.server.service.RetrieveService; import com.tencent.supersonic.headless.server.service.DataSetService; +import com.tencent.supersonic.headless.server.service.RetrieveService; import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -48,27 +48,26 @@ public class RetrieveServiceImpl implements RetrieveService { private static final int RESULT_SIZE = 10; - @Autowired - private DataSetService dataSetService; + @Autowired private DataSetService dataSetService; - @Autowired - private SchemaService schemaService; + @Autowired private SchemaService schemaService; - @Autowired - private KnowledgeBaseService knowledgeBaseService; + @Autowired private KnowledgeBaseService knowledgeBaseService; + + @Autowired private SearchMatchStrategy searchMatchStrategy; - @Autowired - private SearchMatchStrategy searchMatchStrategy; @Override public List retrieve(QueryNLReq queryNLReq) { String queryText = queryNLReq.getQueryText(); // 1.get meta info - SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema(queryNLReq.getDataSetIds()); + SemanticSchema semanticSchemaDb = + schemaService.getSemanticSchema(queryNLReq.getDataSetIds()); List metricsDb = semanticSchemaDb.getMetrics(); final Map dataSetIdToName = semanticSchemaDb.getDataSetIdToName(); Map> modelIdToDataSetIds = - dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser()); + dataSetService.getModelIdToDataSetIds( + new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser()); // 2.detect by segment List originals = knowledgeBaseService.getTerms(queryText, modelIdToDataSetIds); log.debug("hanlp parse result: {}", originals); @@ -84,12 +83,17 @@ public class RetrieveServiceImpl implements RetrieveService { regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue())); // 3.get the most matching data - Optional>> mostSimilarSearchResult = regTextMap.entrySet() - .stream() - .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue())) - .reduce((entry1, entry2) -> - entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length() - ? entry1 : entry2); + Optional>> mostSimilarSearchResult = + regTextMap.entrySet().stream() + .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue())) + .reduce( + (entry1, entry2) -> + entry1.getKey().getDetectSegment().length() + >= entry2.getKey() + .getDetectSegment() + .length() + ? entry1 + : entry2); // 4.optimize the results after the query if (!mostSimilarSearchResult.isPresent()) { @@ -101,30 +105,46 @@ public class RetrieveServiceImpl implements RetrieveService { Set searchResults = new LinkedHashSet(); DataSetInfoStat dataSetInfoStat = NatureHelper.getDataSetStat(originals); - List possibleDataSets = getPossibleDataSets(queryNLReq, originals, dataSetInfoStat, dataSetIds); + List possibleDataSets = + getPossibleDataSets(queryNLReq, originals, dataSetInfoStat, dataSetIds); // 5.1 priority dimension metric - boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDataSets), dataSetIdToName, - searchTextEntry, searchResults); + boolean existMetricAndDimension = + searchMetricAndDimension( + new HashSet<>(possibleDataSets), + dataSetIdToName, + searchTextEntry, + searchResults); // 5.2 process based on dimension values MatchText matchText = searchTextEntry.getKey(); - Map natureToNameMap = getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDataSets)); + Map natureToNameMap = + getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDataSets)); log.debug("possibleDataSets:{},natureToNameMap:{}", possibleDataSets, natureToNameMap); for (Map.Entry natureToNameEntry : natureToNameMap.entrySet()) { - Set searchResultSet = searchDimensionValue(metricsDb, dataSetIdToName, - dataSetInfoStat.getMetricDataSetCount(), existMetricAndDimension, - matchText, natureToNameMap, natureToNameEntry, queryNLReq.getQueryFilters()); + Set searchResultSet = + searchDimensionValue( + metricsDb, + dataSetIdToName, + dataSetInfoStat.getMetricDataSetCount(), + existMetricAndDimension, + matchText, + natureToNameMap, + natureToNameEntry, + queryNLReq.getQueryFilters()); searchResults.addAll(searchResultSet); } return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList()); } - private List getPossibleDataSets(QueryNLReq queryCtx, List originals, - DataSetInfoStat dataSetInfoStat, Set dataSetIds) { + private List getPossibleDataSets( + QueryNLReq queryCtx, + List originals, + DataSetInfoStat dataSetInfoStat, + Set dataSetIds) { if (CollectionUtils.isNotEmpty(dataSetIds)) { return new ArrayList<>(dataSetIds); } @@ -133,8 +153,11 @@ public class RetrieveServiceImpl implements RetrieveService { Long contextDataset = queryCtx.getContextParseInfo().getDataSetId(); - log.debug("possibleDataSets:{},dataSetInfoStat:{},contextDataset:{}", - possibleDataSets, dataSetInfoStat, contextDataset); + log.debug( + "possibleDataSets:{},dataSetInfoStat:{},contextDataset:{}", + possibleDataSets, + dataSetInfoStat, + contextDataset); // If nothing is recognized or only metric are present, then add the contextDataset. if (nothingOrOnlyMetric(dataSetInfoStat)) { @@ -144,18 +167,21 @@ public class RetrieveServiceImpl implements RetrieveService { } private boolean nothingOrOnlyMetric(DataSetInfoStat modelStat) { - return modelStat.getMetricDataSetCount() >= 0 && modelStat.getDimensionDataSetCount() <= 0 - && modelStat.getDimensionValueDataSetCount() <= 0 && modelStat.getDataSetCount() <= 0; + return modelStat.getMetricDataSetCount() >= 0 + && modelStat.getDimensionDataSetCount() <= 0 + && modelStat.getDimensionValueDataSetCount() <= 0 + && modelStat.getDataSetCount() <= 0; } - private Set searchDimensionValue(List metricsDb, - Map modelToName, - long metricModelCount, - boolean existMetricAndDimension, - MatchText matchText, - Map natureToNameMap, - Map.Entry natureToNameEntry, - QueryFilters queryFilters) { + private Set searchDimensionValue( + List metricsDb, + Map modelToName, + long metricModelCount, + boolean existMetricAndDimension, + MatchText matchText, + Map natureToNameMap, + Map.Entry natureToNameEntry, + QueryFilters queryFilters) { Set searchResults = new LinkedHashSet(); String nature = natureToNameEntry.getKey(); @@ -168,13 +194,14 @@ public class RetrieveServiceImpl implements RetrieveService { return searchResults; } // If there are no metric/dimension, complete the metric information - SearchResult searchResult = SearchResult.builder() - .modelId(modelId) - .modelName(modelToName.get(modelId)) - .recommend(matchText.getRegText() + wordName) - .schemaElementType(schemaElementType) - .subRecommend(wordName) - .build(); + SearchResult searchResult = + SearchResult.builder() + .modelId(modelId) + .modelName(modelToName.get(modelId)) + .recommend(matchText.getRegText() + wordName) + .schemaElementType(schemaElementType) + .subRecommend(wordName) + .build(); if (metricModelCount <= 0 && !existMetricAndDimension) { if (filterByQueryFilter(wordName, queryFilters)) { @@ -182,18 +209,24 @@ public class RetrieveServiceImpl implements RetrieveService { } searchResults.add(searchResult); int metricSize = getMetricSize(natureToNameMap); - List metrics = filerMetricsByModel(metricsDb, modelId, metricSize * 3) - .stream() - .limit(metricSize).collect(Collectors.toList()); + List metrics = + filerMetricsByModel(metricsDb, modelId, metricSize * 3).stream() + .limit(metricSize) + .collect(Collectors.toList()); for (String metric : metrics) { - SearchResult result = SearchResult.builder() - .modelId(modelId) - .modelName(modelToName.get(modelId)) - .recommend(matchText.getRegText() + wordName + DictWordType.SPACE + metric) - .subRecommend(wordName + DictWordType.SPACE + metric) - .isComplete(false) - .build(); + SearchResult result = + SearchResult.builder() + .modelId(modelId) + .modelName(modelToName.get(modelId)) + .recommend( + matchText.getRegText() + + wordName + + DictWordType.SPACE + + metric) + .subRecommend(wordName + DictWordType.SPACE + metric) + .isComplete(false) + .build(); searchResults.add(result); } } else { @@ -223,51 +256,67 @@ public class RetrieveServiceImpl implements RetrieveService { return true; } - protected List filerMetricsByModel(List metricsDb, Long model, int metricSize) { + protected List filerMetricsByModel( + List metricsDb, Long model, int metricSize) { if (CollectionUtils.isEmpty(metricsDb)) { return Lists.newArrayList(); } return metricsDb.stream() .filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getDataSetId())) .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .flatMap(entry -> { - List result = new ArrayList<>(); - result.add(entry.getName()); - return result.stream(); - }) - .limit(metricSize).collect(Collectors.toList()); + .flatMap( + entry -> { + List result = new ArrayList<>(); + result.add(entry.getName()); + return result.stream(); + }) + .limit(metricSize) + .collect(Collectors.toList()); } - /*** - * convert nature to name + /** + * * convert nature to name + * * @param recommendTextListEntry * @return */ - private Map getNatureToNameMap(Map.Entry> recommendTextListEntry, - Set possibleModels) { + private Map getNatureToNameMap( + Map.Entry> recommendTextListEntry, + Set possibleModels) { List recommendValues = recommendTextListEntry.getValue(); return recommendValues.stream() - .flatMap(entry -> entry.getNatures().stream() - .filter(nature -> { - if (CollectionUtils.isEmpty(possibleModels)) { - return true; - } - Long model = NatureHelper.getDataSetId(nature); - return possibleModels.contains(model); - }) - .map(nature -> { - DictWord posDO = new DictWord(); - posDO.setWord(entry.getName()); - posDO.setNature(nature); - return posDO; - })).sorted(Comparator.comparingInt(a -> a.getWord().length())) - .collect(Collectors.toMap(DictWord::getNature, DictWord::getWord, (value1, value2) -> value1, - LinkedHashMap::new)); + .flatMap( + entry -> + entry.getNatures().stream() + .filter( + nature -> { + if (CollectionUtils.isEmpty(possibleModels)) { + return true; + } + Long model = NatureHelper.getDataSetId(nature); + return possibleModels.contains(model); + }) + .map( + nature -> { + DictWord posDO = new DictWord(); + posDO.setWord(entry.getName()); + posDO.setNature(nature); + return posDO; + })) + .sorted(Comparator.comparingInt(a -> a.getWord().length())) + .collect( + Collectors.toMap( + DictWord::getNature, + DictWord::getWord, + (value1, value2) -> value1, + LinkedHashMap::new)); } - private boolean searchMetricAndDimension(Set possibleDataSets, Map modelToName, - Map.Entry> searchTextEntry, - Set searchResults) { + private boolean searchMetricAndDimension( + Set possibleDataSets, + Map modelToName, + Map.Entry> searchTextEntry, + Set searchResults) { boolean existMetric = false; log.debug("searchMetricAndDimension searchTextEntry:{}", searchTextEntry); MatchText matchText = searchTextEntry.getKey(); @@ -275,10 +324,15 @@ public class RetrieveServiceImpl implements RetrieveService { for (HanlpMapResult hanlpMapResult : hanlpMapResults) { - List dimensionMetricClassIds = hanlpMapResult.getNatures().stream() - .map(nature -> new ModelWithSemanticType(NatureHelper.getDataSetId(nature), - NatureHelper.convertToElementType(nature))) - .filter(entry -> matchCondition(entry, possibleDataSets)).collect(Collectors.toList()); + List dimensionMetricClassIds = + hanlpMapResult.getNatures().stream() + .map( + nature -> + new ModelWithSemanticType( + NatureHelper.getDataSetId(nature), + NatureHelper.convertToElementType(nature))) + .filter(entry -> matchCondition(entry, possibleDataSets)) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dimensionMetricClassIds)) { continue; @@ -287,26 +341,30 @@ public class RetrieveServiceImpl implements RetrieveService { existMetric = true; Long modelId = modelWithSemanticType.getModel(); SchemaElementType schemaElementType = modelWithSemanticType.getSchemaElementType(); - SearchResult searchResult = SearchResult.builder() - .modelId(modelId) - .modelName(modelToName.get(modelId)) - .recommend(matchText.getRegText() + hanlpMapResult.getName()) - .subRecommend(hanlpMapResult.getName()) - .schemaElementType(schemaElementType) - .build(); - //visibility to filter metrics + SearchResult searchResult = + SearchResult.builder() + .modelId(modelId) + .modelName(modelToName.get(modelId)) + .recommend(matchText.getRegText() + hanlpMapResult.getName()) + .subRecommend(hanlpMapResult.getName()) + .schemaElementType(schemaElementType) + .build(); + // visibility to filter metrics searchResults.add(searchResult); } - log.debug("parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}", hanlpMapResult, - dimensionMetricClassIds, possibleDataSets); + log.debug( + "parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}", + hanlpMapResult, + dimensionMetricClassIds, + possibleDataSets); } log.info("searchMetricAndDimension searchResults:{}", searchResults); return existMetric; } private boolean matchCondition(ModelWithSemanticType entry, Set possibleDataSets) { - if (!(SchemaElementType.METRIC.equals(entry.getSchemaElementType()) || SchemaElementType.DIMENSION.equals( - entry.getSchemaElementType()))) { + if (!(SchemaElementType.METRIC.equals(entry.getSchemaElementType()) + || SchemaElementType.DIMENSION.equals(entry.getSchemaElementType()))) { return false; } @@ -315,5 +373,4 @@ public class RetrieveServiceImpl implements RetrieveService { } return possibleDataSets.contains(entry.getModel()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index fdce37d4f..dc78421ab 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; @@ -36,7 +37,6 @@ import com.tencent.supersonic.headless.api.pojo.response.TermResp; import com.tencent.supersonic.headless.server.manager.DimensionYamlManager; import com.tencent.supersonic.headless.server.manager.MetricYamlManager; import com.tencent.supersonic.headless.server.manager.ModelYamlManager; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; @@ -103,14 +103,17 @@ public class SchemaServiceImpl implements SchemaService { @Value("${s2.schema.cache.enable:true}") private boolean schemaCacheEnable; - public SchemaServiceImpl(ModelService modelService, + public SchemaServiceImpl( + ModelService modelService, DimensionService dimensionService, MetricService metricService, DomainService domainService, DataSetService dataSetService, ModelRelaService modelRelaService, - StatUtils statUtils, TagMetaService tagService, - TermService termService, DatabaseService databaseService) { + StatUtils statUtils, + TagMetaService tagService, + TermService termService, + DatabaseService databaseService) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; @@ -139,7 +142,9 @@ public class SchemaServiceImpl implements SchemaService { if (dataSetId == null) { return null; } - return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst().orElse(null); + return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream() + .findFirst() + .orElse(null); } private List fetchDataSetSchema(Set ids) { @@ -154,8 +159,10 @@ public class SchemaServiceImpl implements SchemaService { ids.add(dataSetId); List dataSetSchemaResps = fetchDataSetSchema(ids); if (!CollectionUtils.isEmpty(dataSetSchemaResps)) { - Optional dataSetSchemaResp = dataSetSchemaResps.stream() - .filter(d -> d.getId().equals(dataSetId)).findFirst(); + Optional dataSetSchemaResp = + dataSetSchemaResps.stream() + .filter(d -> d.getId().equals(dataSetId)) + .findFirst(); if (dataSetSchemaResp.isPresent()) { DataSetSchemaResp dataSetSchema = dataSetSchemaResp.get(); return DataSetSchemaBuilder.build(dataSetSchema); @@ -190,13 +197,15 @@ public class SchemaServiceImpl implements SchemaService { metaFilter.setIds(filter.getDataSetIds()); List dataSetResps = dataSetService.getDataSetList(metaFilter); Map dataSetRespMap = getDataSetMap(dataSetResps); - Set domainIds = dataSetResps.stream().map(DataSetResp::getDomainId) - .collect(Collectors.toSet()); - List modelIds = dataSetRespMap.values().stream().map(DataSetResp::getAllModels) - .flatMap(Collection::stream).collect(Collectors.toList()); + Set domainIds = + dataSetResps.stream().map(DataSetResp::getDomainId).collect(Collectors.toSet()); + List modelIds = + dataSetRespMap.values().stream() + .map(DataSetResp::getAllModels) + .flatMap(Collection::stream) + .collect(Collectors.toList()); Map> termMaps = termService.getTermSets(domainIds); - metaFilter.setModelIds(modelIds); metaFilter.setIds(Lists.newArrayList()); @@ -213,20 +222,30 @@ public class SchemaServiceImpl implements SchemaService { List dataSetSchemaResps = new ArrayList<>(); for (Long dataSetId : dataSetRespMap.keySet()) { DataSetResp dataSetResp = dataSetRespMap.get(dataSetId); - if (dataSetResp == null || !StatusEnum.ONLINE.getCode().equals(dataSetResp.getStatus())) { + if (dataSetResp == null + || !StatusEnum.ONLINE.getCode().equals(dataSetResp.getStatus())) { continue; } - List metricSchemaResps = MetricConverter.filterByDataSet(metricResps, dataSetResp) - .stream().map(this::convert).collect(Collectors.toList()); - List dimSchemaResps = DimensionConverter.filterByDataSet(dimensionResps, dataSetResp) - .stream().map(this::convert).collect(Collectors.toList()); + List metricSchemaResps = + MetricConverter.filterByDataSet(metricResps, dataSetResp).stream() + .map(this::convert) + .collect(Collectors.toList()); + List dimSchemaResps = + DimensionConverter.filterByDataSet(dimensionResps, dataSetResp).stream() + .map(this::convert) + .collect(Collectors.toList()); DataSetSchemaResp dataSetSchemaResp = new DataSetSchemaResp(); BeanUtils.copyProperties(dataSetResp, dataSetSchemaResp); dataSetSchemaResp.setDimensions(dimSchemaResps); dataSetSchemaResp.setMetrics(metricSchemaResps); - dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp -> - dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList())); - dataSetSchemaResp.setTermResps(termMaps.getOrDefault(dataSetResp.getDomainId(), Lists.newArrayList())); + dataSetSchemaResp.setModelResps( + modelResps.stream() + .filter( + modelResp -> + dataSetResp.getAllModels().contains(modelResp.getId())) + .collect(Collectors.toList())); + dataSetSchemaResp.setTermResps( + termMaps.getOrDefault(dataSetResp.getDomainId(), Lists.newArrayList())); dataSetSchemaResps.add(dataSetSchemaResp); } fillStaticInfo(dataSetSchemaResps); @@ -240,10 +259,12 @@ public class SchemaServiceImpl implements SchemaService { } MetaFilter metaFilter = new MetaFilter(modelIds); metaFilter.setStatus(StatusEnum.ONLINE.getCode()); - Map> metricRespMap = metricService.getMetrics(metaFilter) - .stream().collect(Collectors.groupingBy(MetricResp::getModelId)); - Map> dimensionRespsMap = dimensionService.getDimensions(metaFilter) - .stream().collect(Collectors.groupingBy(DimensionResp::getModelId)); + Map> metricRespMap = + metricService.getMetrics(metaFilter).stream() + .collect(Collectors.groupingBy(MetricResp::getModelId)); + Map> dimensionRespsMap = + dimensionService.getDimensions(metaFilter).stream() + .collect(Collectors.groupingBy(DimensionResp::getModelId)); List modelRelas = modelRelaService.getModelRela(modelIds); ModelFilter modelFilter = new ModelFilter(true, modelIds); Map modelMap = modelService.getModelMap(modelFilter); @@ -252,31 +273,43 @@ public class SchemaServiceImpl implements SchemaService { if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) { continue; } - List metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList()); - List metricSchemaResps = metricResps.stream() - .map(this::convert).collect(Collectors.toList()); - List dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()) - .stream().map(this::convert).collect(Collectors.toList()); + List metricResps = + metricRespMap.getOrDefault(modelId, Lists.newArrayList()); + List metricSchemaResps = + metricResps.stream().map(this::convert).collect(Collectors.toList()); + List dimensionResps = + dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()).stream() + .map(this::convert) + .collect(Collectors.toList()); ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); BeanUtils.copyProperties(modelResp, modelSchemaResp); modelSchemaResp.setDimensions(dimensionResps); modelSchemaResp.setMetrics(metricSchemaResps); - modelSchemaResp.setModelRelas(modelRelas.stream().filter(modelRela - -> modelRela.getFromModelId().equals(modelId) || modelRela.getToModelId().equals(modelId)) - .collect(Collectors.toList())); + modelSchemaResp.setModelRelas( + modelRelas.stream() + .filter( + modelRela -> + modelRela.getFromModelId().equals(modelId) + || modelRela.getToModelId().equals(modelId)) + .collect(Collectors.toList())); modelSchemaResps.add(modelSchemaResp); } return modelSchemaResps; - } private void fillCnt(List dataSetSchemaResps, List statInfos) { - Map typeIdAndStatPair = statInfos.stream() - .collect(Collectors.toMap( - itemUseInfo -> itemUseInfo.getType() + AT_SYMBOL + AT_SYMBOL + itemUseInfo.getBizName(), - itemUseInfo -> itemUseInfo, - (item1, item2) -> item1)); + Map typeIdAndStatPair = + statInfos.stream() + .collect( + Collectors.toMap( + itemUseInfo -> + itemUseInfo.getType() + + AT_SYMBOL + + AT_SYMBOL + + itemUseInfo.getBizName(), + itemUseInfo -> itemUseInfo, + (item1, item2) -> item1)); log.debug("typeIdAndStatPair:{}", typeIdAndStatPair); for (DataSetSchemaResp dataSetSchemaResp : dataSetSchemaResps) { fillDimCnt(dataSetSchemaResp, typeIdAndStatPair); @@ -284,37 +317,49 @@ public class SchemaServiceImpl implements SchemaService { } } - private void fillMetricCnt(DataSetSchemaResp dataSetSchemaResp, Map typeIdAndStatPair) { + private void fillMetricCnt( + DataSetSchemaResp dataSetSchemaResp, Map typeIdAndStatPair) { List metrics = dataSetSchemaResp.getMetrics(); if (CollectionUtils.isEmpty(dataSetSchemaResp.getMetrics())) { return; } if (!CollectionUtils.isEmpty(metrics)) { - metrics.stream().forEach(metric -> { - String key = TypeEnums.METRIC.name().toLowerCase() - + AT_SYMBOL + AT_SYMBOL + metric.getBizName(); - if (typeIdAndStatPair.containsKey(key)) { - metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt()); - } - }); + metrics.stream() + .forEach( + metric -> { + String key = + TypeEnums.METRIC.name().toLowerCase() + + AT_SYMBOL + + AT_SYMBOL + + metric.getBizName(); + if (typeIdAndStatPair.containsKey(key)) { + metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt()); + } + }); } dataSetSchemaResp.setMetrics(metrics); } - private void fillDimCnt(DataSetSchemaResp dataSetSchemaResp, Map typeIdAndStatPair) { + private void fillDimCnt( + DataSetSchemaResp dataSetSchemaResp, Map typeIdAndStatPair) { List dimensions = dataSetSchemaResp.getDimensions(); if (CollectionUtils.isEmpty(dataSetSchemaResp.getDimensions())) { return; } if (!CollectionUtils.isEmpty(dimensions)) { - dimensions.stream().forEach(dim -> { - String key = TypeEnums.DIMENSION.name().toLowerCase() - + AT_SYMBOL + AT_SYMBOL + dim.getBizName(); - if (typeIdAndStatPair.containsKey(key)) { - dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt()); - } - }); + dimensions.stream() + .forEach( + dim -> { + String key = + TypeEnums.DIMENSION.name().toLowerCase() + + AT_SYMBOL + + AT_SYMBOL + + dim.getBizName(); + if (typeIdAndStatPair.containsKey(key)) { + dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt()); + } + }); } dataSetSchemaResp.setDimensions(dimensions); } @@ -353,9 +398,11 @@ public class SchemaServiceImpl implements SchemaService { public List getModelList(List modelIds) { List modelRespList = new ArrayList<>(); if (!org.apache.commons.collections.CollectionUtils.isEmpty(modelIds)) { - modelIds.stream().forEach(m -> { - modelRespList.add(modelService.getModel(m)); - }); + modelIds.stream() + .forEach( + m -> { + modelRespList.add(modelService.getModel(m)); + }); } return modelRespList; } @@ -365,7 +412,8 @@ public class SchemaServiceImpl implements SchemaService { semanticSchemaResp.setDataSetId(schemaFilterReq.getDataSetId()); semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds()); if (schemaFilterReq.getDataSetId() != null) { - DataSetSchemaResp dataSetSchemaResp = fetchDataSetSchema(schemaFilterReq.getDataSetId()); + DataSetSchemaResp dataSetSchemaResp = + fetchDataSetSchema(schemaFilterReq.getDataSetId()); BeanUtils.copyProperties(dataSetSchemaResp, semanticSchemaResp); List modelIds = dataSetSchemaResp.getAllModels(); MetaFilter metaFilter = new MetaFilter(); @@ -378,14 +426,25 @@ public class SchemaServiceImpl implements SchemaService { semanticSchemaResp.setModelIds(modelIds); semanticSchemaResp.setSchemaType(SchemaType.DATASET); } else if (!CollectionUtils.isEmpty(schemaFilterReq.getModelIds())) { - List modelSchemaResps = fetchModelSchemaResps(schemaFilterReq.getModelIds()); - semanticSchemaResp.setMetrics(modelSchemaResps.stream().map(ModelSchemaResp::getMetrics) - .flatMap(Collection::stream).collect(Collectors.toList())); - semanticSchemaResp.setDimensions(modelSchemaResps.stream().map(ModelSchemaResp::getDimensions) - .flatMap(Collection::stream).collect(Collectors.toList())); - semanticSchemaResp.setModelRelas(modelSchemaResps.stream().map(ModelSchemaResp::getModelRelas) - .flatMap(Collection::stream).collect(Collectors.toList())); - semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); + List modelSchemaResps = + fetchModelSchemaResps(schemaFilterReq.getModelIds()); + semanticSchemaResp.setMetrics( + modelSchemaResps.stream() + .map(ModelSchemaResp::getMetrics) + .flatMap(Collection::stream) + .collect(Collectors.toList())); + semanticSchemaResp.setDimensions( + modelSchemaResps.stream() + .map(ModelSchemaResp::getDimensions) + .flatMap(Collection::stream) + .collect(Collectors.toList())); + semanticSchemaResp.setModelRelas( + modelSchemaResps.stream() + .map(ModelSchemaResp::getModelRelas) + .flatMap(Collection::stream) + .collect(Collectors.toList())); + semanticSchemaResp.setModelResps( + modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); semanticSchemaResp.setSchemaType(SchemaType.MODEL); } if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { @@ -396,7 +455,8 @@ public class SchemaServiceImpl implements SchemaService { semanticSchemaResp.setTags(tagResps); } if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { - DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); + DatabaseResp databaseResp = + modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); semanticSchemaResp.setDatabaseResp(databaseResp); } return semanticSchemaResp; @@ -419,11 +479,13 @@ public class SchemaServiceImpl implements SchemaService { @Override public List getStatInfo(ItemUseReq itemUseReq) { if (itemUseReq.getCacheEnable()) { - return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> { - List data = statUtils.getStatInfo(itemUseReq); - itemUseCache.put(JsonUtil.toString(itemUseReq), data); - return data; - }); + return itemUseCache.get( + JsonUtil.toString(itemUseReq), + () -> { + List data = statUtils.getStatInfo(itemUseReq); + itemUseCache.put(JsonUtil.toString(itemUseReq), data); + return data; + }); } return statUtils.getStatInfo(itemUseReq); } @@ -431,17 +493,28 @@ public class SchemaServiceImpl implements SchemaService { @Override public List getDomainDataSetTree() { List domainResps = domainService.getDomainList(); - List itemResps = domainResps.stream().map(domain -> - new ItemResp(domain.getId(), domain.getParentId(), domain.getName(), TypeEnums.DOMAIN)) - .collect(Collectors.toList()); - Map itemRespMap = itemResps.stream() - .collect(Collectors.toMap(ItemResp::getId, item -> item)); + List itemResps = + domainResps.stream() + .map( + domain -> + new ItemResp( + domain.getId(), + domain.getParentId(), + domain.getName(), + TypeEnums.DOMAIN)) + .collect(Collectors.toList()); + Map itemRespMap = + itemResps.stream().collect(Collectors.toMap(ItemResp::getId, item -> item)); List dataSetResps = dataSetService.getDataSetList(new MetaFilter()); for (DataSetResp dataSetResp : dataSetResps) { ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId()); if (itemResp != null) { - ItemResp dataSet = new ItemResp(dataSetResp.getId(), dataSetResp.getDomainId(), - dataSetResp.getName(), TypeEnums.DATASET); + ItemResp dataSet = + new ItemResp( + dataSetResp.getId(), + dataSetResp.getDomainId(), + dataSetResp.getName(), + TypeEnums.DATASET); itemResp.getChildren().add(dataSet); } } @@ -449,8 +522,10 @@ public class SchemaServiceImpl implements SchemaService { } private void fillStaticInfo(List dataSetSchemaResps) { - List dataSetIds = dataSetSchemaResps.stream() - .map(DataSetSchemaResp::getId).collect(Collectors.toList()); + List dataSetIds = + dataSetSchemaResps.stream() + .map(DataSetSchemaResp::getId) + .collect(Collectors.toList()); ItemUseReq itemUseReq = new ItemUseReq(); itemUseReq.setModelIds(dataSetIds); @@ -463,8 +538,8 @@ public class SchemaServiceImpl implements SchemaService { if (CollectionUtils.isEmpty(dataSetResps)) { return new HashMap<>(); } - return dataSetResps.stream().collect( - Collectors.toMap(DataSetResp::getId, dataSetResp -> dataSetResp)); + return dataSetResps.stream() + .collect(Collectors.toMap(DataSetResp::getId, dataSetResp -> dataSetResp)); } private DimSchemaResp convert(DimensionResp dimensionResp) { @@ -486,11 +561,12 @@ public class SchemaServiceImpl implements SchemaService { } @Override - public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp, - Map> dimensionYamlMap, - List dataModelYamlTplList, - List metricYamlTplList, - Map modelIdName) { + public void getSchemaYamlTpl( + SemanticSchemaResp semanticSchemaResp, + Map> dimensionYamlMap, + List dataModelYamlTplList, + List metricYamlTplList, + Map modelIdName) { List modelResps = semanticSchemaResp.getModelResps(); if (org.apache.commons.collections.CollectionUtils.isEmpty(modelResps)) { @@ -505,11 +581,16 @@ public class SchemaServiceImpl implements SchemaService { if (!dimensionYamlMap.containsKey(modelResp.getBizName())) { dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>()); } - List dimensionRespList = dimensionResps.stream() - .filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName())) - .collect(Collectors.toList()); - dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml( - dimensionRespList)); + List dimensionRespList = + dimensionResps.stream() + .filter( + d -> + d.getModelBizName() + .equalsIgnoreCase(modelResp.getBizName())) + .collect(Collectors.toList()); + dimensionYamlMap + .get(modelResp.getBizName()) + .addAll(DimensionYamlManager.convert2DimensionYaml(dimensionRespList)); } List metricResps = new ArrayList<>(semanticSchemaResp.getMetrics()); metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps)); @@ -524,5 +605,4 @@ public class SchemaServiceImpl implements SchemaService { public DatabaseResp getDatabase(Long id) { return databaseService.getDatabase(id); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java index a2e7b5c0f..5c2beb538 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java @@ -24,13 +24,13 @@ import com.tencent.supersonic.headless.server.persistence.repository.TagReposito import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.pojo.TagObjectFilter; +import com.tencent.supersonic.headless.server.service.CollectService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.TagMetaService; import com.tencent.supersonic.headless.server.service.TagObjectService; -import com.tencent.supersonic.headless.server.service.CollectService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -58,10 +58,14 @@ public class TagMetaServiceImpl implements TagMetaService { private final TagObjectService tagObjectService; private final DomainService domainService; - public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService, - CollectService collectService, @Lazy DimensionService dimensionService, - @Lazy MetricService metricService, TagObjectService tagObjectService, - DomainService domainService) { + public TagMetaServiceImpl( + TagRepository tagRepository, + ModelService modelService, + CollectService collectService, + @Lazy DimensionService dimensionService, + @Lazy MetricService metricService, + TagObjectService tagObjectService, + DomainService domainService) { this.tagRepository = tagRepository; this.modelService = modelService; this.collectService = collectService; @@ -156,22 +160,34 @@ public class TagMetaServiceImpl implements TagMetaService { } if (Objects.nonNull(tagMarketPageReq.getTagObjectId())) { - modelRespList = modelRespList.stream() - .filter(modelResp -> tagMarketPageReq.getTagObjectId().equals(modelResp.getTagObjectId())) - .collect(Collectors.toList()); + modelRespList = + modelRespList.stream() + .filter( + modelResp -> + tagMarketPageReq + .getTagObjectId() + .equals(modelResp.getTagObjectId())) + .collect(Collectors.toList()); } if (CollectionUtils.isEmpty(modelRespList)) { return new PageInfo(); } - List modelIds = modelRespList.stream().map(model -> model.getId()).collect(Collectors.toList()); + List modelIds = + modelRespList.stream().map(model -> model.getId()).collect(Collectors.toList()); TagFilter tagFilter = new TagFilter(); BeanUtils.copyProperties(tagMarketPageReq, tagFilter); List collectList = collectService.getCollectionList(user.getName()); if (tagMarketPageReq.isHasCollect()) { - List collectIds = collectList.stream() - .filter(collectDO -> SchemaElementType.TAG.name().equalsIgnoreCase(collectDO.getType())) - .map(CollectDO::getCollectId).collect(Collectors.toList()); + List collectIds = + collectList.stream() + .filter( + collectDO -> + SchemaElementType.TAG + .name() + .equalsIgnoreCase(collectDO.getType())) + .map(CollectDO::getCollectId) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(collectIds)) { tagFilter.setIds(Lists.newArrayList(-1L)); } else { @@ -179,9 +195,9 @@ public class TagMetaServiceImpl implements TagMetaService { } } tagFilter.setModelIds(modelIds); - PageInfo tagDOPageInfo = PageHelper.startPage(tagMarketPageReq.getCurrent(), - tagMarketPageReq.getPageSize()) - .doSelectPageInfo(() -> getTags(tagFilter)); + PageInfo tagDOPageInfo = + PageHelper.startPage(tagMarketPageReq.getCurrent(), tagMarketPageReq.getPageSize()) + .doSelectPageInfo(() -> getTags(tagFilter)); List tagRespList = tagDOPageInfo.getList(); if (CollectionUtils.isEmpty(tagRespList)) { @@ -201,14 +217,22 @@ public class TagMetaServiceImpl implements TagMetaService { if (CollectionUtils.isEmpty(tagObjects)) { return; } - Map tagObjectMap = tagObjects.stream() - .collect(Collectors.toMap(TagObjectResp::getId, tagObject -> tagObject, (v1, v2) -> v2)); + Map tagObjectMap = + tagObjects.stream() + .collect( + Collectors.toMap( + TagObjectResp::getId, + tagObject -> tagObject, + (v1, v2) -> v2)); if (CollectionUtils.isNotEmpty(tagRespList)) { - tagRespList.stream().forEach(tagResp -> { - if (tagObjectMap.containsKey(tagResp.getTagObjectId())) { - tagResp.setTagObjectName(tagObjectMap.get(tagResp.getTagObjectId()).getName()); - } - }); + tagRespList.stream() + .forEach( + tagResp -> { + if (tagObjectMap.containsKey(tagResp.getTagObjectId())) { + tagResp.setTagObjectName( + tagObjectMap.get(tagResp.getTagObjectId()).getName()); + } + }); } } @@ -222,14 +246,20 @@ public class TagMetaServiceImpl implements TagMetaService { } private void fillDomainInfo(List tagRespList) { - Map domainMap = domainService.getDomainList().stream() - .collect(Collectors.toMap(DomainResp::getId, domain -> domain, (v1, v2) -> v2)); + Map domainMap = + domainService.getDomainList().stream() + .collect( + Collectors.toMap( + DomainResp::getId, domain -> domain, (v1, v2) -> v2)); if (CollectionUtils.isNotEmpty(tagRespList) && Objects.nonNull(domainMap)) { - tagRespList.stream().forEach(tagResp -> { - if (domainMap.containsKey(tagResp.getDomainId())) { - tagResp.setDomainName(domainMap.get(tagResp.getDomainId()).getName()); - } - }); + tagRespList.stream() + .forEach( + tagResp -> { + if (domainMap.containsKey(tagResp.getDomainId())) { + tagResp.setDomainName( + domainMap.get(tagResp.getDomainId()).getName()); + } + }); } } @@ -282,29 +312,40 @@ public class TagMetaServiceImpl implements TagMetaService { } } modelRespList.add(modelResp); - } return modelRespList; } private void fillModelInfo(List tagRespList) { - List modelIds = tagRespList.stream().map(TagResp::getModelId) - .collect(Collectors.toList()); + List modelIds = + tagRespList.stream().map(TagResp::getModelId).collect(Collectors.toList()); ModelFilter modelFilter = new ModelFilter(false, modelIds); Map modelIdAndRespMap = modelService.getModelMap(modelFilter); - tagRespList.stream().forEach(tagResp -> { - if (Objects.nonNull(modelIdAndRespMap) && modelIdAndRespMap.containsKey(tagResp.getModelId())) { - tagResp.setModelName(modelIdAndRespMap.get(tagResp.getModelId()).getName()); - tagResp.setDomainId(modelIdAndRespMap.get(tagResp.getModelId()).getDomainId()); - tagResp.setTagObjectId(modelIdAndRespMap.get(tagResp.getModelId()).getTagObjectId()); - } - }); + tagRespList.stream() + .forEach( + tagResp -> { + if (Objects.nonNull(modelIdAndRespMap) + && modelIdAndRespMap.containsKey(tagResp.getModelId())) { + tagResp.setModelName( + modelIdAndRespMap.get(tagResp.getModelId()).getName()); + tagResp.setDomainId( + modelIdAndRespMap.get(tagResp.getModelId()).getDomainId()); + tagResp.setTagObjectId( + modelIdAndRespMap + .get(tagResp.getModelId()) + .getTagObjectId()); + } + }); } private TagResp fillCollectAndAdminInfo(TagResp tagResp, User user) { - List collectIds = collectService.getCollectionList(user.getName()) - .stream().filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType())) - .map(CollectDO::getCollectId).collect(Collectors.toList()); + List collectIds = + collectService.getCollectionList(user.getName()).stream() + .filter( + collectDO -> + TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType())) + .map(CollectDO::getCollectId) + .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) { tagResp.setIsCollect(true); } else { @@ -316,30 +357,40 @@ public class TagMetaServiceImpl implements TagMetaService { } private TagResp fillCollectAndAdminInfo(List tagRespList, User user) { - List collectIds = collectService.getCollectionList(user.getName()) - .stream().filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType())) - .map(CollectDO::getCollectId).collect(Collectors.toList()); + List collectIds = + collectService.getCollectionList(user.getName()).stream() + .filter( + collectDO -> + TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType())) + .map(CollectDO::getCollectId) + .collect(Collectors.toList()); - tagRespList.stream().forEach(tagResp -> { - if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) { - tagResp.setIsCollect(true); - } else { - tagResp.setIsCollect(false); - } - }); + tagRespList.stream() + .forEach( + tagResp -> { + if (CollectionUtils.isNotEmpty(collectIds) + && collectIds.contains(tagResp.getId())) { + tagResp.setIsCollect(true); + } else { + tagResp.setIsCollect(false); + } + }); fillAdminRes(tagRespList, user); return tagRespList.get(0); } private void fillAdminRes(List tagRespList, User user) { - List modelRespList = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); + List modelRespList = + modelService.getModelListWithAuth(user, null, AuthType.ADMIN); if (CollectionUtils.isEmpty(modelRespList)) { return; } - Set modelIdSet = modelRespList.stream().map(ModelResp::getId).collect(Collectors.toSet()); + Set modelIdSet = + modelRespList.stream().map(ModelResp::getId).collect(Collectors.toSet()); for (TagResp tagResp : tagRespList) { - if (modelIdSet.contains(tagResp.getModelId()) || tagResp.getCreatedBy().equalsIgnoreCase(user.getName())) { + if (modelIdSet.contains(tagResp.getModelId()) + || tagResp.getCreatedBy().equalsIgnoreCase(user.getName())) { tagResp.setHasAdminRes(true); } else { tagResp.setHasAdminRes(false); @@ -356,7 +407,8 @@ public class TagMetaServiceImpl implements TagMetaService { List tagRespList = tagRepository.getTagDOList(tagFilter); if (!CollectionUtils.isEmpty(tagRespList)) { - throw new RuntimeException(String.format("the tag is exit, itemId:%s", tagReq.getItemId())); + throw new RuntimeException( + String.format("the tag is exit, itemId:%s", tagReq.getItemId())); } } @@ -365,16 +417,22 @@ public class TagMetaServiceImpl implements TagMetaService { DimensionResp dimension = dimensionService.getDimension(tagReq.getItemId()); ModelResp model = modelService.getModel(dimension.getModelId()); if (Objects.isNull(model.getTagObjectId())) { - throw new RuntimeException(String.format("this dimension:%s is not supported to create tag," - + " no related tag object", tagReq.getItemId())); + throw new RuntimeException( + String.format( + "this dimension:%s is not supported to create tag," + + " no related tag object", + tagReq.getItemId())); } } if (TagDefineType.METRIC.equals(tagReq.getTagDefineType())) { MetricResp metric = metricService.getMetric(tagReq.getItemId()); ModelResp model = modelService.getModel(metric.getModelId()); if (Objects.isNull(model.getTagObjectId())) { - throw new RuntimeException(String.format("this metric:%s is not supported to create tag," - + " no related tag object", tagReq.getItemId())); + throw new RuntimeException( + String.format( + "this metric:%s is not supported to create tag," + + " no related tag object", + tagReq.getItemId())); } } } @@ -391,14 +449,17 @@ public class TagMetaServiceImpl implements TagMetaService { TagFilter tagFilter = new TagFilter(); tagFilter.setTagDefineType(tagDefineType); tagFilter.setItemIds(itemIds); - Set dimensionItemSet = getTagDOList(tagFilter).stream().map(TagDO::getItemId) - .collect(Collectors.toSet()); - return itemIds.stream().map(entry -> { - TagItem tagItem = new TagItem(); - tagItem.setIsTag(Boolean.compare(dimensionItemSet.contains(entry), false)); - tagItem.setItemId(entry); - return tagItem; - } - ).collect(Collectors.toList()); + Set dimensionItemSet = + getTagDOList(tagFilter).stream().map(TagDO::getItemId).collect(Collectors.toSet()); + return itemIds.stream() + .map( + entry -> { + TagItem tagItem = new TagItem(); + tagItem.setIsTag( + Boolean.compare(dimensionItemSet.contains(entry), false)); + tagItem.setItemId(entry); + return tagItem; + }) + .collect(Collectors.toList()); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagObjectServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagObjectServiceImpl.java index 787bcf187..96aa360e8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagObjectServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagObjectServiceImpl.java @@ -34,8 +34,10 @@ public class TagObjectServiceImpl implements TagObjectService { private final ModelService modelService; private final TagMetaService tagMetaService; - public TagObjectServiceImpl(TagObjectRepository tagObjectRepository, ModelService modelService, - @Lazy TagMetaService tagMetaService) { + public TagObjectServiceImpl( + TagObjectRepository tagObjectRepository, + ModelService modelService, + @Lazy TagMetaService tagMetaService) { this.tagObjectRepository = tagObjectRepository; this.modelService = modelService; this.tagMetaService = tagMetaService; @@ -64,12 +66,18 @@ public class TagObjectServiceImpl implements TagObjectService { if (CollectionUtils.isEmpty(tagObjectRespList)) { return; } - tagObjectRespList = tagObjectRespList.stream() - .filter(tagObjectResp -> StatusEnum.ONLINE.getCode().equals(tagObjectResp.getStatus())) - .collect(Collectors.toList()); + tagObjectRespList = + tagObjectRespList.stream() + .filter( + tagObjectResp -> + StatusEnum.ONLINE + .getCode() + .equals(tagObjectResp.getStatus())) + .collect(Collectors.toList()); for (TagObjectResp tagObject : tagObjectRespList) { if (tagObject.getBizName().equalsIgnoreCase(tagObjectReq.getBizName())) { - throw new Exception(String.format("the bizName %s is exist", tagObjectReq.getBizName())); + throw new Exception( + String.format("the bizName %s is exist", tagObjectReq.getBizName())); } if (tagObject.getName().equalsIgnoreCase(tagObjectReq.getName())) { throw new Exception(String.format("the name %s is exist", tagObjectReq.getName())); @@ -115,11 +123,14 @@ public class TagObjectServiceImpl implements TagObjectService { } private void checkTagObjectStatus(TagObjectDO tagObjectDO, User user) throws Exception { - List allModelByDomainIds = modelService.getAllModelByDomainIds( - Arrays.asList(tagObjectDO.getDomainId())); + List allModelByDomainIds = + modelService.getAllModelByDomainIds(Arrays.asList(tagObjectDO.getDomainId())); if (!CollectionUtils.isEmpty(allModelByDomainIds)) { - List modelIds = allModelByDomainIds.stream().map(ModelResp::getId).collect(Collectors.toList()); - throw new Exception("delete operation is not supported at the moment. related modelIds:" + modelIds); + List modelIds = + allModelByDomainIds.stream().map(ModelResp::getId).collect(Collectors.toList()); + throw new Exception( + "delete operation is not supported at the moment. related modelIds:" + + modelIds); } TagFilterPageReq tagMarketPageReq = new TagFilterPageReq(); tagMarketPageReq.setTagObjectId(tagObjectDO.getId()); @@ -127,18 +138,21 @@ public class TagObjectServiceImpl implements TagObjectService { if (Objects.nonNull(respPageInfo)) { List tagRespList = respPageInfo.getList(); if (!CollectionUtils.isEmpty(tagRespList)) { - List tagIds = tagRespList.stream().map(TagResp::getId).collect(Collectors.toList()); - throw new Exception("delete operation is not supported at the moment. related tagIds:" + tagIds); + List tagIds = + tagRespList.stream().map(TagResp::getId).collect(Collectors.toList()); + throw new Exception( + "delete operation is not supported at the moment. related tagIds:" + + tagIds); } } - } private void checkDeletePermission(TagObjectDO tagObjectDO, User user) throws Exception { if (user.getName().equalsIgnoreCase(tagObjectDO.getCreatedBy()) || user.isSuperAdmin()) { return; } - throw new Exception("delete operation is not supported at the moment. Please contact the admin."); + throw new Exception( + "delete operation is not supported at the moment. Please contact the admin."); } @Override @@ -157,9 +171,11 @@ public class TagObjectServiceImpl implements TagObjectService { public Map getAllTagObjectMap() { TagObjectFilter filter = new TagObjectFilter(); List tagObjectDOList = tagObjectRepository.query(filter); - List tagObjectRespList = TagObjectConverter.convert2RespList(tagObjectDOList); + List tagObjectRespList = + TagObjectConverter.convert2RespList(tagObjectDOList); Map map = - tagObjectRespList.stream().collect(Collectors.toMap(TagObjectResp::getId, a -> a, (k1, k2) -> k1)); + tagObjectRespList.stream() + .collect(Collectors.toMap(TagObjectResp::getId, a -> a, (k1, k2) -> k1)); return map; } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java index 98ba8cda4..1d84a2851 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java @@ -14,10 +14,10 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.TagMetaService; import com.tencent.supersonic.headless.server.service.TagQueryService; -import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -50,8 +50,11 @@ public class TagQueryServiceImpl implements TagQueryService { private final ModelService modelService; private final SqlGenerateUtils sqlGenerateUtils; - public TagQueryServiceImpl(TagMetaService tagMetaService, SemanticLayerService queryService, - ModelService modelService, SqlGenerateUtils sqlGenerateUtils) { + public TagQueryServiceImpl( + TagMetaService tagMetaService, + SemanticLayerService queryService, + ModelService modelService, + SqlGenerateUtils sqlGenerateUtils) { this.tagMetaService = tagMetaService; this.queryService = queryService; this.modelService = modelService; @@ -81,19 +84,24 @@ public class TagQueryServiceImpl implements TagQueryService { } private void checkTag(TagResp tag) throws Exception { - if (Objects.nonNull(tag) && TagDefineType.METRIC.name().equalsIgnoreCase(tag.getTagDefineType())) { - throw new Exception("do not support value distribution query for tag (from metric): " + tag.getBizName()); + if (Objects.nonNull(tag) + && TagDefineType.METRIC.name().equalsIgnoreCase(tag.getTagDefineType())) { + throw new Exception( + "do not support value distribution query for tag (from metric): " + + tag.getBizName()); } } - private void correctDateConf(ItemValueReq itemValueReq, TagResp tag, User user) throws Exception { + private void correctDateConf(ItemValueReq itemValueReq, TagResp tag, User user) + throws Exception { ModelResp model = modelService.getModel(tag.getModelId()); List timeDimension = model.getTimeDimension(); if (CollectionUtils.isEmpty(timeDimension)) { itemValueReq.setDateConf(null); return; } - if (Objects.nonNull(itemValueReq.getDateConf()) && itemValueReq.getDateConf().getUnit() == 1) { + if (Objects.nonNull(itemValueReq.getDateConf()) + && itemValueReq.getDateConf().getUnit() == 1) { return; } @@ -111,10 +119,16 @@ public class TagQueryServiceImpl implements TagQueryService { return LocalDate.now().plusDays(-dayBefore).format(formatter); } - private String queryTagDateFromDbBySql(Dim dim, TagResp tag, ItemValueReq itemValueReq, User user) { + private String queryTagDateFromDbBySql( + Dim dim, TagResp tag, ItemValueReq itemValueReq, User user) { String sqlPattern = "select max(%s) as %s from tbl where %s is not null"; - String sql = String.format(sqlPattern, TimeDimensionEnum.DAY.getName(), maxDateAlias, tag.getBizName()); + String sql = + String.format( + sqlPattern, + TimeDimensionEnum.DAY.getName(), + maxDateAlias, + tag.getBizName()); // 添加时间过滤信息 log.info("[queryTagDateFromDbBySql] calculate the maximum time start"); @@ -127,12 +141,22 @@ public class TagQueryServiceImpl implements TagQueryService { if (StringUtils.isEmpty(dateFormat)) { dateFormat = itemValueDateFormat; } - String start = LocalDate.now().minusDays(itemValueReq.getDateConf().getUnit()) - .format(DateTimeFormatter.ofPattern(dateFormat)); - String end = LocalDate.now().minusDays(0) - .format(DateTimeFormatter.ofPattern(dateFormat)); - sql = sql + String.format(" and ( %s > '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), - start, TimeDimensionEnum.DAY.getName(), end); + String start = + LocalDate.now() + .minusDays(itemValueReq.getDateConf().getUnit()) + .format(DateTimeFormatter.ofPattern(dateFormat)); + String end = + LocalDate.now() + .minusDays(0) + .format(DateTimeFormatter.ofPattern(dateFormat)); + sql = + sql + + String.format( + " and ( %s > '%s' and %s <= '%s' )", + TimeDimensionEnum.DAY.getName(), + start, + TimeDimensionEnum.DAY.getName(), + end); } } } @@ -158,10 +182,10 @@ public class TagQueryServiceImpl implements TagQueryService { String dateDefault = queryTagDate(dim); log.info("queryTagDate by default, dateDefault:{}.", dateDefault); return dateDefault; - } - private Long queryTagTotalCount(TagResp tag, ItemValueReq itemValueReq, User user) throws Exception { + private Long queryTagTotalCount(TagResp tag, ItemValueReq itemValueReq, User user) + throws Exception { String sqlPattern = "select count(1) as %s from tbl where %s is not null %s"; String dateFilter = getDateFilter(itemValueReq); String sql = String.format(sqlPattern, tagValueAlias, tag.getBizName(), dateFilter); @@ -188,32 +212,47 @@ public class TagQueryServiceImpl implements TagQueryService { if (Objects.isNull(itemValueReq.getDateConf())) { return ""; } - String dateWhereClause = sqlGenerateUtils.getDateWhereClause(itemValueReq.getDateConf(), null); + String dateWhereClause = + sqlGenerateUtils.getDateWhereClause(itemValueReq.getDateConf(), null); return " and " + dateWhereClause; } - private void fillTagValueInfo(ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) { + private void fillTagValueInfo( + ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) { List valueDistributionList = new ArrayList<>(); List> resultList = semanticQueryResp.getResultList(); if (!CollectionUtils.isEmpty(resultList)) { - resultList.stream().forEach(line -> { - Object tagValue = line.get(itemValueResp.getBizName()); - Long tagValueCount = Long.parseLong(line.get(tagValueAlias).toString()); - valueDistributionList.add(ValueDistribution.builder() - .totalCount(totalCount) - .valueMap(tagValue) - .valueCount(tagValueCount) - .ratio(1.0 * tagValueCount / totalCount).build()); - }); + resultList.stream() + .forEach( + line -> { + Object tagValue = line.get(itemValueResp.getBizName()); + Long tagValueCount = + Long.parseLong(line.get(tagValueAlias).toString()); + valueDistributionList.add( + ValueDistribution.builder() + .totalCount(totalCount) + .valueMap(tagValue) + .valueCount(tagValueCount) + .ratio(1.0 * tagValueCount / totalCount) + .build()); + }); } itemValueResp.setValueDistributionList(valueDistributionList); } private QuerySqlReq generateReq(TagResp tag, ItemValueReq itemValueReq) { - String sqlPattern = "select %s, count(1) as %s from tbl where %s is not null %s " - + "group by %s order by %s desc"; - String sql = String.format(sqlPattern, tag.getBizName(), tagValueAlias, tag.getBizName(), - getDateFilter(itemValueReq), tag.getBizName(), tag.getBizName()); + String sqlPattern = + "select %s, count(1) as %s from tbl where %s is not null %s " + + "group by %s order by %s desc"; + String sql = + String.format( + sqlPattern, + tag.getBizName(), + tagValueAlias, + tag.getBizName(), + getDateFilter(itemValueReq), + tag.getBizName(), + tag.getBizName()); Set modelIds = new HashSet<>(); modelIds.add(tag.getModelId()); @@ -233,4 +272,4 @@ public class TagQueryServiceImpl implements TagQueryService { } return dateConf; } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TermServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TermServiceImpl.java index 8076bb382..9280c355d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TermServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TermServiceImpl.java @@ -19,7 +19,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - @Service public class TermServiceImpl extends ServiceImpl implements TermService { @@ -58,8 +57,9 @@ public class TermServiceImpl extends ServiceImpl implements QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().in(TermDO::getDomainId, domainIds); List list = list(queryWrapper); - return list.stream().map(this::convert).collect( - Collectors.groupingBy(TermResp::getDomainId)); + return list.stream() + .map(this::convert) + .collect(Collectors.groupingBy(TermResp::getDomainId)); } private TermResp convert(TermDO termDO) { @@ -77,5 +77,4 @@ public class TermServiceImpl extends ServiceImpl implements termDO.setRelatedDimensions(JsonUtil.toString(termReq.getRelateDimensions())); termDO.setRelatedMetrics(JsonUtil.toString(termReq.getRelatedMetrics())); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java index 6eb564792..397b41854 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java @@ -13,8 +13,7 @@ import org.springframework.stereotype.Component; @Order(2) public class DictionaryReloadTask implements CommandLineRunner { - @Autowired - private DictWordService dictWordService; + @Autowired private DictWordService dictWordService; @Override public void run(String... args) { @@ -31,9 +30,7 @@ public class DictionaryReloadTask implements CommandLineRunner { } } - /*** - * reload knowledge task - */ + /** * reload knowledge task */ @Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}") public void reloadKnowledge() { log.debug("reloadKnowledge start"); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java index 33cabf3b2..cb1f44167 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.headless.server.task; +import javax.annotation.PreDestroy; + import com.tencent.supersonic.headless.server.service.FlightService; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.flight.FlightServer; import org.apache.arrow.flight.Location; @@ -12,25 +12,29 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; -import javax.annotation.PreDestroy; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; -/** - * Initialize flight jdbc server - */ +/** Initialize flight jdbc server */ @Component @Slf4j public class FlightServerInitTask implements CommandLineRunner { @Value("${s2.flightSql.enable:false}") private Boolean enable = false; + @Value("${s2.flightSql.host:localhost}") private String host = "localhost"; + @Value("${s2.flightSql.port:9081}") private Integer port = 9081; + @Value("${s2.flightSql.executor:4}") private Integer executor = 4; + @Value("${s2.flightSql.queue:128}") private Integer queue = 128; + @Value("${s2.flightSql.expireMinute:10}") private Integer expireMinute = 10; @@ -47,8 +51,7 @@ public class FlightServerInitTask implements CommandLineRunner { executorService = Executors.newFixedThreadPool(executor); this.flightService.setExecutorService(executorService, queue, expireMinute); Location listenLocation = Location.forGrpcInsecure(host, port); - flightServer = FlightServer.builder(allocator, listenLocation, this.flightService) - .build(); + flightServer = FlightServer.builder(allocator, listenLocation, this.flightService).build(); } public String getHost() { @@ -67,7 +70,6 @@ public class FlightServerInitTask implements CommandLineRunner { } catch (Exception e) { log.error("FlightServerInitTask start error {}", e); } - } public Boolean isRunning() { @@ -93,15 +95,18 @@ public class FlightServerInitTask implements CommandLineRunner { public void run() { try { startServer(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - flightServer.close(); - allocator.close(); - } catch (Exception e) { - log.error("flightServer close error {}", e); - } - })); - //flightServer.awaitTermination(); + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + try { + flightServer.close(); + allocator.close(); + } catch (Exception e) { + log.error("flightServer close error {}", e); + } + })); + // flightServer.awaitTermination(); } catch (Exception e) { log.error("run error {}", e); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java index 5f063612a..ab334dfaa 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.task; +import javax.annotation.PreDestroy; + import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.service.EmbeddingService; @@ -16,7 +18,6 @@ import org.springframework.core.annotation.Order; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import javax.annotation.PreDestroy; import java.util.List; @Component @@ -24,17 +25,13 @@ import java.util.List; @Order(2) public class MetaEmbeddingTask implements CommandLineRunner { - @Autowired - private EmbeddingService embeddingService; + @Autowired private EmbeddingService embeddingService; - @Autowired - private EmbeddingConfig embeddingConfig; + @Autowired private EmbeddingConfig embeddingConfig; - @Autowired - private MetricService metricService; + @Autowired private MetricService metricService; - @Autowired - private DimensionService dimensionService; + @Autowired private DimensionService dimensionService; @PreDestroy public void onShutdown() { @@ -58,20 +55,20 @@ public class MetaEmbeddingTask implements CommandLineRunner { embeddingStorePersistFile(); } - /*** - * reload meta embedding - */ + /** * reload meta embedding */ @Scheduled(cron = "${s2.reload.meta.embedding.corn:0 0 */2 * * ?}") public void reloadMetaEmbedding() { long startTime = System.currentTimeMillis(); try { List metricDataItems = metricService.getDataEvent().getDataItems(); - embeddingService.addQuery(embeddingConfig.getMetaCollectionName(), + embeddingService.addQuery( + embeddingConfig.getMetaCollectionName(), TextSegmentConvert.convertToEmbedding(metricDataItems)); List dimensionDataItems = dimensionService.getDataEvent().getDataItems(); - embeddingService.addQuery(embeddingConfig.getMetaCollectionName(), + embeddingService.addQuery( + embeddingConfig.getMetaCollectionName(), TextSegmentConvert.convertToEmbedding(dimensionDataItems)); } catch (Exception e) { log.error("Failed to reload meta embedding.", e); @@ -88,4 +85,4 @@ public class MetaEmbeddingTask implements CommandLineRunner { log.error("initMetaEmbedding error", e); } } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java index 368269449..429c1bf8b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.utils; - import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import dev.langchain4j.data.message.AiMessage; @@ -18,46 +17,44 @@ import org.springframework.stereotype.Component; import java.util.HashMap; import java.util.Map; - @Component @Slf4j public class AliasGenerateHelper { private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); - private static final String NAME_ALIAS_INSTRUCTION = "" - + "\n#Role: You are a professional data analyst specializing in metrics and dimensions." - + "\n#Task: You will be provided with metadata about a metric or dimension, please help " - + "generate a few aliases in the same language as its `fieldName`." - + "\n#Rules:" - + "1. Please do not generate aliases like xxx1, xxx2, xxx3." - + "2. Please do not generate aliases that are the same as the original names of metrics/dimensions." - + "3. Please pay attention to the quality of the generated aliases and " - + "avoid creating aliases that look like test data." - + "4. Please output as a json string array." - + "\n#Metadata: {'table':'{{table}}', 'name':'{{name}}', 'type':'{{type}}', " - + "'field':'field', 'description':'{{desc}}'}" - + "\n#Output:"; + private static final String NAME_ALIAS_INSTRUCTION = + "" + + "\n#Role: You are a professional data analyst specializing in metrics and dimensions." + + "\n#Task: You will be provided with metadata about a metric or dimension, please help " + + "generate a few aliases in the same language as its `fieldName`." + + "\n#Rules:" + + "1. Please do not generate aliases like xxx1, xxx2, xxx3." + + "2. Please do not generate aliases that are the same as the original names of metrics/dimensions." + + "3. Please pay attention to the quality of the generated aliases and " + + "avoid creating aliases that look like test data." + + "4. Please output as a json string array." + + "\n#Metadata: {'table':'{{table}}', 'name':'{{name}}', 'type':'{{type}}', " + + "'field':'field', 'description':'{{desc}}'}" + + "\n#Output:"; - private static final String VALUE_ALIAS_INSTRUCTION = "" - + "\n#Role: You are a professional data analyst." - + "\n#Task: You will be provided with a json array of dimension values," - + "please help generate a few aliases for each value." - + "\n#Rule:" - + "1. ALWAYS output json array for each value." - + "2. The aliases should be in the same language as its original value." - + "\n#Exemplar:" - + "Values: [\\\"qq_music\\\",\\\"kugou_music\\\"], " - + "Output: {\\\"tran\\\":[\\\"qq音乐\\\",\\\"酷狗音乐\\\"]," - + " \\\"alias\\\":{\\\"qq_music\\\":[\\\"q音\\\",\\\"qq音乐\\\"]," - + " \\\"kugou_music\\\":[\\\"kugou\\\",\\\"酷狗\\\"]}}" - + "\nValues: {{values}}, Output:"; + private static final String VALUE_ALIAS_INSTRUCTION = + "" + + "\n#Role: You are a professional data analyst." + + "\n#Task: You will be provided with a json array of dimension values," + + "please help generate a few aliases for each value." + + "\n#Rule:" + + "1. ALWAYS output json array for each value." + + "2. The aliases should be in the same language as its original value." + + "\n#Exemplar:" + + "Values: [\\\"qq_music\\\",\\\"kugou_music\\\"], " + + "Output: {\\\"tran\\\":[\\\"qq音乐\\\",\\\"酷狗音乐\\\"]," + + " \\\"alias\\\":{\\\"qq_music\\\":[\\\"q音\\\",\\\"qq音乐\\\"]," + + " \\\"kugou_music\\\":[\\\"kugou\\\",\\\"酷狗\\\"]}}" + + "\nValues: {{values}}, Output:"; - public String generateAlias(String mockType, - String name, - String bizName, - String table, - String desc) { + public String generateAlias( + String mockType, String name, String bizName, String table, String desc) { Map variable = new HashMap<>(); variable.put("table", table); variable.put("name", name); @@ -91,7 +88,8 @@ public class AliasGenerateHelper { return response.content().text(); } - private static String extractString(String targetString, String left, String right, Boolean exclusionFlag) { + private static String extractString( + String targetString, String left, String right, Boolean exclusionFlag) { if (targetString == null || left == null || right == null || exclusionFlag == null) { return targetString; } @@ -104,7 +102,8 @@ public class AliasGenerateHelper { if (secondIndex == -1) { return null; } - String extractedString = targetString.substring(firstIndex + left.length(), secondIndex); + String extractedString = + targetString.substring(firstIndex + left.length(), secondIndex); if (!exclusionFlag) { extractedString = left + extractedString + right; } @@ -132,6 +131,7 @@ public class AliasGenerateHelper { final String left; final String right; final Boolean exclusionFlag; + public BoundaryPattern(String start, String end, Boolean includeMarkers) { this.left = start; this.right = end; @@ -139,36 +139,37 @@ public class AliasGenerateHelper { } } BoundaryPattern[] patterns = { - //不做任何匹配 - new BoundaryPattern(null, null, null), - //```{"name":"Alice","age":25,"city":"NewYork"}``` - new BoundaryPattern("```", "```", true), - //```json {"name":"Alice","age":25,"city":"NewYork"}``` - new BoundaryPattern("```json", "```", true), - //```JSON {"name":"Alice","age":25,"city":"NewYork"}``` - new BoundaryPattern("```JSON", "```", true), - //{"name":"Alice","age":25,"city":"NewYork"} - new BoundaryPattern("{", "}", false), - //["Alice", "Bob"] - new BoundaryPattern("[", "]", false) + // 不做任何匹配 + new BoundaryPattern(null, null, null), + // ```{"name":"Alice","age":25,"city":"NewYork"}``` + new BoundaryPattern("```", "```", true), + // ```json {"name":"Alice","age":25,"city":"NewYork"}``` + new BoundaryPattern("```json", "```", true), + // ```JSON {"name":"Alice","age":25,"city":"NewYork"}``` + new BoundaryPattern("```JSON", "```", true), + // {"name":"Alice","age":25,"city":"NewYork"} + new BoundaryPattern("{", "}", false), + // ["Alice", "Bob"] + new BoundaryPattern("[", "]", false) }; for (BoundaryPattern pattern : patterns) { - String extracted = extractString(aiMessage, pattern.left, pattern.right, pattern.exclusionFlag); + String extracted = + extractString(aiMessage, pattern.left, pattern.right, pattern.exclusionFlag); if (extracted == null) { continue; } - //判断是否能解析为Object或者Array + // 判断是否能解析为Object或者Array try { JSON.parseObject(extracted); return extracted; } catch (JSONException ignored) { - //ignored + // ignored } try { JSON.parseArray(extracted); return extracted; } catch (JSONException ignored) { - //ignored + // ignored } } throw new JSONException("json extract failed"); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java index 46ca010d2..37008e74d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java @@ -34,7 +34,8 @@ public class ChatWorkflowEngine { private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); private final List schemaMappers = ComponentFactory.getSchemaMappers(); private final List semanticParsers = ComponentFactory.getSemanticParsers(); - private final List semanticCorrectors = ComponentFactory.getSemanticCorrectors(); + private final List semanticCorrectors = + ComponentFactory.getSemanticCorrectors(); private final List resultProcessors = ComponentFactory.getResultProcessors(); public void execute(ChatQueryContext queryCtx, ParseResp parseResult) { @@ -45,7 +46,8 @@ public class ChatWorkflowEngine { performMapping(queryCtx); if (queryCtx.getMapInfo().getMatchedDataSetInfos().isEmpty()) { parseResult.setState(ParseResp.ParseState.FAILED); - parseResult.setErrorMsg("No semantic entities can be mapped against user question."); + parseResult.setErrorMsg( + "No semantic entities can be mapped against user question."); queryCtx.setChatWorkflowState(ChatWorkflowState.FINISHED); } else if (queryCtx.getMapInfo().needContinueMap()) { queryCtx.setChatWorkflowState(ChatWorkflowState.MAPPING); @@ -60,8 +62,10 @@ public class ChatWorkflowEngine { parseResult.setErrorMsg("No semantic queries can be parsed out."); queryCtx.setChatWorkflowState(ChatWorkflowState.FINISHED); } else { - List parseInfos = queryCtx.getCandidateQueries().stream() - .map(SemanticQuery::getParseInfo).collect(Collectors.toList()); + List parseInfos = + queryCtx.getCandidateQueries().stream() + .map(SemanticQuery::getParseInfo) + .collect(Collectors.toList()); parseResult.setSelectedParses(parseInfos); queryCtx.setChatWorkflowState(ChatWorkflowState.CORRECTING); } @@ -97,10 +101,14 @@ public class ChatWorkflowEngine { } private void performParsing(ChatQueryContext queryCtx) { - semanticParsers.forEach(parser -> { - parser.parse(queryCtx); - log.debug("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx)); - }); + semanticParsers.forEach( + parser -> { + parser.parse(queryCtx); + log.debug( + "{} result:{}", + parser.getClass().getSimpleName(), + JsonUtil.toString(queryCtx)); + }); } private void performCorrecting(ChatQueryContext queryCtx) { @@ -118,36 +126,45 @@ public class ChatWorkflowEngine { } private void performProcessing(ChatQueryContext queryCtx, ParseResp parseResult) { - resultProcessors.forEach(processor -> { - processor.process(parseResult, queryCtx); - }); + resultProcessors.forEach( + processor -> { + processor.process(parseResult, queryCtx); + }); } private void performTranslating(ChatQueryContext chatQueryContext) { - List semanticParseInfos = chatQueryContext.getCandidateQueries().stream() - .map(SemanticQuery::getParseInfo) - .collect(Collectors.toList()); + List semanticParseInfos = + chatQueryContext.getCandidateQueries().stream() + .map(SemanticQuery::getParseInfo) + .collect(Collectors.toList()); - semanticParseInfos.forEach(parseInfo -> { - try { - SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); - if (Objects.isNull(semanticQuery)) { - return; - } - semanticQuery.setParseInfo(parseInfo); - SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); - SemanticLayerService queryService = ContextUtils.getBean(SemanticLayerService.class); - SemanticTranslateResp explain = queryService.translate(semanticQueryReq, chatQueryContext.getUser()); - parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL()); + semanticParseInfos.forEach( + parseInfo -> { + try { + SemanticQuery semanticQuery = + QueryManager.createQuery(parseInfo.getQueryMode()); + if (Objects.isNull(semanticQuery)) { + return; + } + semanticQuery.setParseInfo(parseInfo); + SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); + SemanticLayerService queryService = + ContextUtils.getBean(SemanticLayerService.class); + SemanticTranslateResp explain = + queryService.translate( + semanticQueryReq, chatQueryContext.getUser()); + parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL()); - keyPipelineLog.info("SqlInfoProcessor results:\n" - + "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}", - StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()), - StringUtils.normalizeSpace(parseInfo.getSqlInfo().getCorrectedS2SQL()), - StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL())); - } catch (Exception e) { - log.warn("get sql info failed:{}", parseInfo, e); - } - }); + keyPipelineLog.info( + "SqlInfoProcessor results:\n" + + "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}", + StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()), + StringUtils.normalizeSpace( + parseInfo.getSqlInfo().getCorrectedS2SQL()), + StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL())); + } catch (Exception e) { + log.warn("get sql info failed:{}", parseInfo, e); + } + }); } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ClassConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ClassConverter.java index bcaff7a15..482f51a26 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ClassConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ClassConverter.java @@ -29,8 +29,10 @@ public class ClassConverter { private final DomainService domainService; private final TagObjectService tagObjectService; - public ClassConverter(ClassRepository classRepository, DomainService domainService, - TagObjectService tagObjectService) { + public ClassConverter( + ClassRepository classRepository, + DomainService domainService, + TagObjectService tagObjectService) { this.classRepository = classRepository; this.domainService = domainService; this.tagObjectService = tagObjectService; @@ -55,13 +57,16 @@ public class ClassConverter { return convert2RespInternal(classDO, idAndDomain, classFullPathMap); } - private ClassResp convert2RespInternal(ClassDO classDO, Map idAndDomain, - Map classFullPathMap) { + private ClassResp convert2RespInternal( + ClassDO classDO, + Map idAndDomain, + Map classFullPathMap) { ClassResp classResp = new ClassResp(); BeanUtils.copyProperties(classDO, classResp); Long domainId = classResp.getDomainId(); - if (Objects.nonNull(idAndDomain) && idAndDomain.containsKey(domainId) + if (Objects.nonNull(idAndDomain) + && idAndDomain.containsKey(domainId) && Objects.nonNull(idAndDomain.get(domainId))) { classResp.setDomainName(idAndDomain.get(domainId).getName()); } @@ -89,8 +94,9 @@ public class ClassConverter { public Map getClassFullPathMap() { Map classFullPathMap = new HashMap<>(); List classDOList = classRepository.getAllClassDOList(); - Map classDOMap = classDOList.stream() - .collect(Collectors.toMap(ClassDO::getId, a -> a, (k1, k2) -> k1)); + Map classDOMap = + classDOList.stream() + .collect(Collectors.toMap(ClassDO::getId, a -> a, (k1, k2) -> k1)); for (ClassDO classDO : classDOList) { final Long domainId = classDO.getId(); StringBuilder fullPath = new StringBuilder(classDO.getBizName() + "/"); @@ -116,5 +122,4 @@ public class ClassConverter { public Map getIdAndTagSet() { return tagObjectService.getAllTagObjectMap(); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java index 4a6451ef3..2322b2090 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java @@ -12,9 +12,7 @@ import org.springframework.core.io.support.SpringFactoriesLoader; import java.util.ArrayList; import java.util.List; -/** - * QueryConverter QueryOptimizer QueryExecutor object factory - */ +/** QueryConverter QueryOptimizer QueryExecutor object factory */ @Slf4j public class ComponentFactory { private static List resultProcessors = new ArrayList<>(); @@ -23,21 +21,27 @@ public class ComponentFactory { private static List semanticCorrectors = new ArrayList<>(); public static List getResultProcessors() { - return CollectionUtils.isEmpty(resultProcessors) ? init(ResultProcessor.class, - resultProcessors) : resultProcessors; + return CollectionUtils.isEmpty(resultProcessors) + ? init(ResultProcessor.class, resultProcessors) + : resultProcessors; } public static List getSchemaMappers() { - return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers) : schemaMappers; + return CollectionUtils.isEmpty(schemaMappers) + ? init(SchemaMapper.class, schemaMappers) + : schemaMappers; } public static List getSemanticParsers() { - return CollectionUtils.isEmpty(semanticParsers) ? init(SemanticParser.class, semanticParsers) : semanticParsers; + return CollectionUtils.isEmpty(semanticParsers) + ? init(SemanticParser.class, semanticParsers) + : semanticParsers; } public static List getSemanticCorrectors() { - return CollectionUtils.isEmpty(semanticCorrectors) ? init(SemanticCorrector.class, - semanticCorrectors) : semanticCorrectors; + return CollectionUtils.isEmpty(semanticCorrectors) + ? init(SemanticCorrector.class, semanticCorrectors) + : semanticCorrectors; } public static T getBean(String name, Class tClass) { @@ -45,14 +49,15 @@ public class ComponentFactory { } private static List init(Class factoryType, List list) { - list.addAll(SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader())); + list.addAll( + SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader())); return list; } private static T init(Class factoryType) { - return SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader()).get(0); + return SpringFactoriesLoader.loadFactories( + factoryType, Thread.currentThread().getContextClassLoader()) + .get(0); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 6c9713520..065fbce6e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -31,14 +31,15 @@ public class DataSetSchemaBuilder { public static DataSetSchema build(DataSetSchemaResp resp) { DataSetSchema dataSetSchema = new DataSetSchema(); dataSetSchema.setQueryConfig(resp.getQueryConfig()); - SchemaElement dataSet = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .id(resp.getId()) - .name(resp.getName()) - .bizName(resp.getBizName()) - .type(SchemaElementType.DATASET) - .build(); + SchemaElement dataSet = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .id(resp.getId()) + .name(resp.getName()) + .bizName(resp.getBizName()) + .type(SchemaElementType.DATASET) + .build(); dataSetSchema.setDataSet(dataSet); Set metrics = getMetrics(resp); @@ -71,20 +72,21 @@ public class DataSetSchemaBuilder { for (MetricSchemaResp metric : resp.getMetrics()) { List alias = SchemaItem.getAliasList(metric.getAlias()); if (metric.getIsTag() == 1) { - SchemaElement tagToAdd = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .model(metric.getModelId()) - .id(metric.getId()) - .name(metric.getName()) - .bizName(metric.getBizName()) - .type(SchemaElementType.TAG) - .useCnt(metric.getUseCnt()) - .alias(alias) - .defaultAgg(metric.getDefaultAgg()) - .isTag(metric.getIsTag()) - .description(metric.getDescription()) - .build(); + SchemaElement tagToAdd = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .model(metric.getModelId()) + .id(metric.getId()) + .name(metric.getName()) + .bizName(metric.getBizName()) + .type(SchemaElementType.TAG) + .useCnt(metric.getUseCnt()) + .alias(alias) + .defaultAgg(metric.getDefaultAgg()) + .isTag(metric.getIsTag()) + .description(metric.getDescription()) + .build(); tags.add(tagToAdd); } } @@ -105,20 +107,21 @@ public class DataSetSchemaBuilder { } } if (dim.getIsTag() == 1) { - SchemaElement tagToAdd = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .model(dim.getModelId()) - .id(dim.getId()) - .name(dim.getName()) - .bizName(dim.getBizName()) - .type(SchemaElementType.TAG) - .useCnt(dim.getUseCnt()) - .alias(alias) - .schemaValueMaps(schemaValueMaps) - .isTag(dim.getIsTag()) - .description(dim.getDescription()) - .build(); + SchemaElement tagToAdd = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .model(dim.getModelId()) + .id(dim.getId()) + .name(dim.getName()) + .bizName(dim.getBizName()) + .type(SchemaElementType.TAG) + .useCnt(dim.getUseCnt()) + .alias(alias) + .schemaValueMaps(schemaValueMaps) + .isTag(dim.getIsTag()) + .description(dim.getDescription()) + .build(); tags.add(tagToAdd); } } @@ -155,24 +158,26 @@ public class DataSetSchemaBuilder { schemaValueMaps.add(schemaValueMap); } } - SchemaElement dimToAdd = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .model(dim.getModelId()) - .id(dim.getId()) - .name(dim.getName()) - .bizName(dim.getBizName()) - .useCnt(dim.getUseCnt()) - .alias(alias) - .schemaValueMaps(schemaValueMaps) - .isTag(dim.getIsTag()) - .description(dim.getDescription()) - .type(SchemaElementType.DIMENSION) - .build(); + SchemaElement dimToAdd = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .model(dim.getModelId()) + .id(dim.getId()) + .name(dim.getName()) + .bizName(dim.getBizName()) + .useCnt(dim.getUseCnt()) + .alias(alias) + .schemaValueMaps(schemaValueMaps) + .isTag(dim.getIsTag()) + .description(dim.getDescription()) + .type(SchemaElementType.DIMENSION) + .build(); dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType()); if (dim.isTimeDimension()) { - String timeFormat = String.valueOf(dim.getExt().get(DimensionConstants.DIMENSION_TIME_FORMAT)); + String timeFormat = + String.valueOf(dim.getExt().get(DimensionConstants.DIMENSION_TIME_FORMAT)); dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat); } dimensions.add(dimToAdd); @@ -195,19 +200,22 @@ public class DataSetSchemaBuilder { } } } - SchemaElement dimValueToAdd = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .model(dim.getModelId()) - .id(dim.getId()) - .name(dim.getName()) - .bizName(dim.getBizName()) - .type(SchemaElementType.VALUE) - .useCnt(dim.getUseCnt()) - .alias(new ArrayList<>(Arrays.asList(dimValueAlias.toArray(new String[0])))) - .isTag(dim.getIsTag()) - .description(dim.getDescription()) - .build(); + SchemaElement dimValueToAdd = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .model(dim.getModelId()) + .id(dim.getId()) + .name(dim.getName()) + .bizName(dim.getBizName()) + .type(SchemaElementType.VALUE) + .useCnt(dim.getUseCnt()) + .alias( + new ArrayList<>( + Arrays.asList(dimValueAlias.toArray(new String[0])))) + .isTag(dim.getIsTag()) + .description(dim.getDescription()) + .build(); dimensionValues.add(dimValueToAdd); } return dimensionValues; @@ -219,24 +227,24 @@ public class DataSetSchemaBuilder { List alias = SchemaItem.getAliasList(metric.getAlias()); - SchemaElement metricToAdd = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .model(metric.getModelId()) - .id(metric.getId()) - .name(metric.getName()) - .bizName(metric.getBizName()) - .type(SchemaElementType.METRIC) - .useCnt(metric.getUseCnt()) - .alias(alias) - .relatedSchemaElements(getRelateSchemaElement(metric)) - .defaultAgg(metric.getDefaultAgg()) - .dataFormatType(metric.getDataFormatType()) - .isTag(metric.getIsTag()) - .description(metric.getDescription()) - .build(); + SchemaElement metricToAdd = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .model(metric.getModelId()) + .id(metric.getId()) + .name(metric.getName()) + .bizName(metric.getBizName()) + .type(SchemaElementType.METRIC) + .useCnt(metric.getUseCnt()) + .alias(alias) + .relatedSchemaElements(getRelateSchemaElement(metric)) + .defaultAgg(metric.getDefaultAgg()) + .dataFormatType(metric.getDataFormatType()) + .isTag(metric.getIsTag()) + .description(metric.getDescription()) + .build(); metrics.add(metricToAdd); - } return metrics; } @@ -245,34 +253,38 @@ public class DataSetSchemaBuilder { Set terms = new HashSet<>(); for (TermResp termResp : resp.getTermResps()) { List alias = termResp.getAlias(); - SchemaElement metricToAdd = SchemaElement.builder() - .dataSetId(resp.getId()) - .dataSetName(resp.getName()) - .model(-1L) - .id(termResp.getId()) - .name(termResp.getName()) - .bizName(termResp.getName()) - .type(SchemaElementType.TERM) - .useCnt(0L) - .alias(alias) - .description(termResp.getDescription()) - .build(); + SchemaElement metricToAdd = + SchemaElement.builder() + .dataSetId(resp.getId()) + .dataSetName(resp.getName()) + .model(-1L) + .id(termResp.getId()) + .name(termResp.getName()) + .bizName(termResp.getName()) + .type(SchemaElementType.TERM) + .useCnt(0L) + .alias(alias) + .description(termResp.getDescription()) + .build(); terms.add(metricToAdd); - } return terms; } - private static List getRelateSchemaElement(MetricSchemaResp metricSchemaResp) { + private static List getRelateSchemaElement( + MetricSchemaResp metricSchemaResp) { RelateDimension relateDimension = metricSchemaResp.getRelateDimension(); - if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { + if (relateDimension == null + || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { return Lists.newArrayList(); } - return relateDimension.getDrillDownDimensions().stream().map(dimension -> { - RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement(); - BeanUtils.copyProperties(dimension, relateSchemaElement); - return relateSchemaElement; - }).collect(Collectors.toList()); + return relateDimension.getDrillDownDimensions().stream() + .map( + dimension -> { + RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement(); + BeanUtils.copyProperties(dimension, relateSchemaElement); + return relateSchemaElement; + }) + .collect(Collectors.toList()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java index ae09d5ad2..95acf699f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java @@ -3,11 +3,12 @@ package com.tencent.supersonic.headless.server.utils; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; import com.tencent.supersonic.headless.core.pojo.ConnectInfo; import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; + import java.util.Arrays; public class DatabaseConverter { @@ -84,5 +85,4 @@ public class DatabaseConverter { connectInfo.setDatabase(databaseReq.getDatabase()); return connectInfo; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 065116a6a..2ebc9ac28 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -76,6 +76,7 @@ public class DictUtils { // 前多少天 @Value("${s2.item.value.date.start:1}") private Integer itemValueDateStart; + @Value("${s2.item.value.date.end:1}") // 前多少天 private Integer itemValueDateEnd; @@ -83,18 +84,18 @@ public class DictUtils { @Value("${s2.item.value.date.format:yyyy-MM-dd}") private String itemValueDateFormat; - private final DimensionService dimensionService; private final MetricService metricService; private final SemanticLayerService queryService; private final ModelService modelService; private final TagMetaService tagMetaService; - public DictUtils(DimensionService dimensionService, - MetricService metricService, - SemanticLayerService queryService, - ModelService modelService, - @Lazy TagMetaService tagMetaService) { + public DictUtils( + DimensionService dimensionService, + MetricService metricService, + SemanticLayerService queryService, + ModelService modelService, + @Lazy TagMetaService tagMetaService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; @@ -103,11 +104,13 @@ public class DictUtils { } public String fetchDictFileName(DictItemResp dictItemResp) { - return String.format("dic_value_%d_%s_%s", dictItemResp.getModelId(), dictItemResp.getType().name(), - dictItemResp.getItemId()); + return String.format( + "dic_value_%d_%s_%s", + dictItemResp.getModelId(), dictItemResp.getType().name(), dictItemResp.getItemId()); } - public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user, TaskStatusEnum status) { + public DictTaskDO generateDictTaskDO( + DictItemResp dictItemResp, User user, TaskStatusEnum status) { DictTaskDO taskDO = new DictTaskDO(); Date createAt = new Date(); String name = dictItemResp.fetchDictFileName(); @@ -117,7 +120,8 @@ public class DictUtils { taskDO.setConfig(JsonUtil.toString(dictItemResp.getConfig())); taskDO.setStatus(status.getStatus()); taskDO.setCreatedAt(createAt); - String creator = (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) ? "" : user.getName(); + String creator = + (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) ? "" : user.getName(); taskDO.setCreatedBy(creator); return taskDO; } @@ -137,13 +141,14 @@ public class DictUtils { public List dictDOList2Req(List dictConfDOList) { List dictItemReqList = new ArrayList<>(); - dictConfDOList.stream().forEach(conf -> { - DictItemResp dictItemResp = dictDO2Req(conf); - if (Objects.nonNull(dictItemResp)) { - dictItemReqList.add(dictDO2Req(conf)); - } - - }); + dictConfDOList.stream() + .forEach( + conf -> { + DictItemResp dictItemResp = dictDO2Req(conf); + if (Objects.nonNull(dictItemResp)) { + dictItemReqList.add(dictDO2Req(conf)); + } + }); return dictItemReqList; } @@ -178,14 +183,17 @@ public class DictUtils { String bizName = dictItemResp.getBizName(); try { SemanticQueryResp semanticQueryResp = queryService.queryByReq(semanticQueryReq, null); - if (Objects.isNull(semanticQueryResp) || CollectionUtils.isEmpty(semanticQueryResp.getResultList())) { + if (Objects.isNull(semanticQueryResp) + || CollectionUtils.isEmpty(semanticQueryResp.getResultList())) { return lines; } Map valueAndFrequencyPair = new HashMap<>(2000); for (Map line : semanticQueryResp.getResultList()) { - if (CollectionUtils.isEmpty(line) || !line.containsKey(bizName) - || line.get(bizName) == null || line.size() != 2) { + if (CollectionUtils.isEmpty(line) + || !line.containsKey(bizName) + || line.get(bizName) == null + || line.size() != 2) { continue; } String dimValue = line.get(bizName).toString(); @@ -210,33 +218,38 @@ public class DictUtils { } private void addWhiteValueLines(DictItemResp dictItemResp, List lines, String nature) { - if (Objects.isNull(dictItemResp) || Objects.isNull(dictItemResp.getConfig()) + if (Objects.isNull(dictItemResp) + || Objects.isNull(dictItemResp.getConfig()) || CollectionUtils.isEmpty(dictItemResp.getConfig().getWhiteList())) { return; } List whiteList = dictItemResp.getConfig().getWhiteList(); - whiteList.forEach(white -> { - if (!StringUtils.isEmpty(white)) { - white = white.replace(SPACE, POUND); - } - lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency)); - }); + whiteList.forEach( + white -> { + if (!StringUtils.isEmpty(white)) { + white = white.replace(SPACE, POUND); + } + lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency)); + }); } - private void constructDictLines(Map valueAndFrequencyPair, List lines, String nature) { + private void constructDictLines( + Map valueAndFrequencyPair, List lines, String nature) { if (CollectionUtils.isEmpty(valueAndFrequencyPair)) { return; } - valueAndFrequencyPair.forEach((value, frequency) -> { - if (!StringUtils.isEmpty(value)) { - value = value.replace(SPACE, POUND); - } - lines.add(String.format("%s %s %s", value, nature, frequency)); - }); + valueAndFrequencyPair.forEach( + (value, frequency) -> { + if (!StringUtils.isEmpty(value)) { + value = value.replace(SPACE, POUND); + } + lines.add(String.format("%s %s %s", value, nature, frequency)); + }); } - private void mergeMultivaluedValue(Map valueAndFrequencyPair, String dimValue, Long metric) { + private void mergeMultivaluedValue( + Map valueAndFrequencyPair, String dimValue, Long metric) { if (StringUtils.isEmpty(dimValue)) { return; } @@ -249,7 +262,10 @@ public class DictUtils { } for (String value : tmp.keySet()) { - long metricOld = valueAndFrequencyPair.containsKey(value) ? valueAndFrequencyPair.get(value) : 0L; + long metricOld = + valueAndFrequencyPair.containsKey(value) + ? valueAndFrequencyPair.get(value) + : 0L; valueAndFrequencyPair.put(value, metric + metricOld); } } @@ -269,13 +285,16 @@ public class DictUtils { String whereStr = generateWhereStr(dictItemResp); String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr; ItemValueConfig config = dictItemResp.getConfig(); - Long limit = (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount : - dictItemResp.getConfig().getLimit(); + Long limit = + (Objects.isNull(config) || Objects.isNull(config.getLimit())) + ? itemValueMaxCount + : dictItemResp.getConfig().getLimit(); // todo 自定义指标 Set modelIds = new HashSet<>(); String metric = "count(1)"; - if (Objects.nonNull(dictItemResp.getConfig()) && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { + if (Objects.nonNull(dictItemResp.getConfig()) + && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { Long metricId = dictItemResp.getConfig().getMetricId(); MetricResp metricResp = metricService.getMetric(metricId); String metricBizName = metricResp.getBizName(); @@ -293,7 +312,8 @@ public class DictUtils { } private QuerySqlReq constructDimQueryReq(DictItemResp dictItemResp) { - if (Objects.nonNull(dictItemResp) && Objects.nonNull(dictItemResp.getConfig()) + if (Objects.nonNull(dictItemResp) + && Objects.nonNull(dictItemResp.getConfig()) && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { // 查询默认指标 QueryStructReq queryStructReq = generateQueryStruct(dictItemResp); @@ -305,13 +325,16 @@ public class DictUtils { private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { - String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d"; + String sqlPattern = + "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d"; String bizName = dictItemResp.getBizName(); String whereStr = generateWhereStr(dictItemResp); String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr; ItemValueConfig config = dictItemResp.getConfig(); - Long limit = (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount : - dictItemResp.getConfig().getLimit(); + Long limit = + (Objects.isNull(config) || Objects.isNull(config.getLimit())) + ? itemValueMaxCount + : dictItemResp.getConfig().getLimit(); String sql = String.format(sqlPattern, bizName, where, bizName, limit); Set modelIds = new HashSet<>(); modelIds.add(dictItemResp.getModelId()); @@ -348,8 +371,10 @@ public class DictUtils { fillStructDateInfo(queryStructReq, dictItemResp); - Long limit = Objects.isNull(dictItemResp.getConfig().getLimit()) ? itemValueMaxCount : - dictItemResp.getConfig().getLimit(); + Long limit = + Objects.isNull(dictItemResp.getConfig().getLimit()) + ? itemValueMaxCount + : dictItemResp.getConfig().getLimit(); queryStructReq.setLimit(limit); queryStructReq.setNeedAuth(false); return queryStructReq; @@ -385,18 +410,25 @@ public class DictUtils { } } - private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model, - Integer itemValueDateStart, Integer itemValueDateEnd) { + private void fillStructDateBetween( + QueryStructReq queryStructReq, + ModelResp model, + Integer itemValueDateStart, + Integer itemValueDateEnd) { if (Objects.nonNull(model)) { List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { DateConf dateConf = new DateConf(); dateConf.setDateMode(DateConf.DateMode.BETWEEN); String format = timeDims.get(0).getDateFormat(); - String start = LocalDate.now().minusDays(itemValueDateStart) - .format(DateTimeFormatter.ofPattern(format)); - String end = LocalDate.now().minusDays(itemValueDateEnd) - .format(DateTimeFormatter.ofPattern(format)); + String start = + LocalDate.now() + .minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(format)); + String end = + LocalDate.now() + .minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(format)); dateConf.setStartDate(start); dateConf.setEndDate(end); queryStructReq.setDateInfo(dateConf); @@ -426,7 +458,8 @@ public class DictUtils { if (Objects.nonNull(config)) { if (!CollectionUtils.isEmpty(config.getBlackList())) { StringJoiner joinerBlack = new StringJoiner(COMMA); - config.getBlackList().stream().forEach(black -> joinerBlack.add(APOSTROPHE + black + APOSTROPHE)); + config.getBlackList().stream() + .forEach(black -> joinerBlack.add(APOSTROPHE + black + APOSTROPHE)); joiner.add(String.format("(%s not in (%s))", bizName, joinerBlack.toString())); } @@ -444,12 +477,17 @@ public class DictUtils { public String defaultDateFilter() { String format = itemValueDateFormat; - String start = LocalDate.now().minusDays(itemValueDateStart) - .format(DateTimeFormatter.ofPattern(format)); - String end = LocalDate.now().minusDays(itemValueDateEnd) - .format(DateTimeFormatter.ofPattern(format)); - return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), start, - TimeDimensionEnum.DAY.getName(), end); + String start = + LocalDate.now() + .minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(format)); + String end = + LocalDate.now() + .minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(format)); + return String.format( + "( %s >= '%s' and %s <= '%s' )", + TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(), end); } private String generateDictDateFilter(DictItemResp dictItemResp) { @@ -464,7 +502,8 @@ public class DictUtils { } // 静态日期 if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) { - return String.format("( %s >= '%s' and %s <= '%s' )", + return String.format( + "( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), config.getDateConf().getStartDate(), TimeDimensionEnum.DAY.getName(), @@ -487,12 +526,20 @@ public class DictUtils { if (StringUtils.isEmpty(dateFormat)) { dateFormat = itemValueDateFormat; } - String start = LocalDate.now().minusDays(dictItemResp.getConfig().getDateConf().getUnit()) - .format(DateTimeFormatter.ofPattern(dateFormat)); - String end = LocalDate.now().minusDays(0) - .format(DateTimeFormatter.ofPattern(dateFormat)); - return String.format("( %s > '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), start, - TimeDimensionEnum.DAY.getName(), end); + String start = + LocalDate.now() + .minusDays(dictItemResp.getConfig().getDateConf().getUnit()) + .format(DateTimeFormatter.ofPattern(dateFormat)); + String end = + LocalDate.now() + .minusDays(0) + .format(DateTimeFormatter.ofPattern(dateFormat)); + return String.format( + "( %s > '%s' and %s <= '%s' )", + TimeDimensionEnum.DAY.getName(), + start, + TimeDimensionEnum.DAY.getName(), + end); } } return ""; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index 01a5649df..880d9ba1b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -65,21 +65,27 @@ public class DimensionConverter { return dimensionDO; } - public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO, - Map modelRespMap) { + public static DimensionResp convert2DimensionResp( + DimensionDO dimensionDO, Map modelRespMap) { DimensionResp dimensionResp = new DimensionResp(); BeanUtils.copyProperties(dimensionDO, dimensionResp); dimensionResp.setModelName( modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getName()); dimensionResp.setModelBizName( - modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getBizName()); + modelRespMap + .getOrDefault(dimensionResp.getModelId(), new ModelResp()) + .getBizName()); if (dimensionDO.getDefaultValues() != null) { - dimensionResp.setDefaultValues(JSONObject.parseObject(dimensionDO.getDefaultValues(), List.class)); + dimensionResp.setDefaultValues( + JSONObject.parseObject(dimensionDO.getDefaultValues(), List.class)); } dimensionResp.setModelFilterSql( - modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getFilterSql()); + modelRespMap + .getOrDefault(dimensionResp.getModelId(), new ModelResp()) + .getFilterSql()); if (StringUtils.isNotEmpty(dimensionDO.getDimValueMaps())) { - dimensionResp.setDimValueMaps(JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class)); + dimensionResp.setDimValueMaps( + JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class)); } if (StringUtils.isNotEmpty(dimensionDO.getDataType())) { dimensionResp.setDataType(DataTypeEnums.of(dimensionDO.getDataType())); @@ -99,7 +105,7 @@ public class DimensionConverter { private static DimensionType getType(String type) { try { - //Support compatibility with legacy data. + // Support compatibility with legacy data. IdentifyType.valueOf(type.toLowerCase()); return DimensionType.identify; } catch (IllegalArgumentException e) { @@ -107,11 +113,15 @@ public class DimensionConverter { } } - public static List filterByDataSet(List dimensionResps, DataSetResp dataSetResp) { - return dimensionResps.stream().filter(dimensionResp -> - dataSetResp.dimensionIds().contains(dimensionResp.getId()) - || dataSetResp.getAllIncludeAllModels().contains(dimensionResp.getModelId())) + public static List filterByDataSet( + List dimensionResps, DataSetResp dataSetResp) { + return dimensionResps.stream() + .filter( + dimensionResp -> + dataSetResp.dimensionIds().contains(dimensionResp.getId()) + || dataSetResp + .getAllIncludeAllModels() + .contains(dimensionResp.getModelId())) .collect(Collectors.toList()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java index a9e7b4a2d..25265fb32 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.utils; - import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; @@ -30,19 +29,26 @@ public class DomainConvert { DomainResp domainResp = new DomainResp(); BeanUtils.copyProperties(domainDO, domainResp); domainResp.setFullPath(domainFullPathMap.get(domainDO.getId())); - domainResp.setAdmins(StringUtils.isBlank(domainDO.getAdmin()) - ? Lists.newArrayList() : Arrays.asList(domainDO.getAdmin().split(","))); - domainResp.setAdminOrgs(StringUtils.isBlank(domainDO.getAdminOrg()) - ? Lists.newArrayList() : Arrays.asList(domainDO.getAdminOrg().split(","))); - domainResp.setViewers(StringUtils.isBlank(domainDO.getViewer()) - ? Lists.newArrayList() : Arrays.asList(domainDO.getViewer().split(","))); - domainResp.setViewOrgs(StringUtils.isBlank(domainDO.getViewOrg()) - ? Lists.newArrayList() : Arrays.asList(domainDO.getViewOrg().split(","))); + domainResp.setAdmins( + StringUtils.isBlank(domainDO.getAdmin()) + ? Lists.newArrayList() + : Arrays.asList(domainDO.getAdmin().split(","))); + domainResp.setAdminOrgs( + StringUtils.isBlank(domainDO.getAdminOrg()) + ? Lists.newArrayList() + : Arrays.asList(domainDO.getAdminOrg().split(","))); + domainResp.setViewers( + StringUtils.isBlank(domainDO.getViewer()) + ? Lists.newArrayList() + : Arrays.asList(domainDO.getViewer().split(","))); + domainResp.setViewOrgs( + StringUtils.isBlank(domainDO.getViewOrg()) + ? Lists.newArrayList() + : Arrays.asList(domainDO.getViewOrg().split(","))); return domainResp; } public static DomainResp convert(DomainDO domainDO) { return convert(domainDO, new HashMap<>()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/FlightUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/FlightUtils.java index e185f0a64..ffe95956e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/FlightUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/FlightUtils.java @@ -5,9 +5,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.regex.Pattern; -/** - * tools for arrow flight sql - */ +/** tools for arrow flight sql */ public class FlightUtils { public static int resolveType(Object value) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java index 5c5e532e6..d71f3f151 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.server.utils; -import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -58,7 +58,8 @@ public class MetricCheckUtils { } String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(metricReq.getName()); if (StringUtils.isNotBlank(forbiddenCharacters)) { - throw new InvalidArgumentException(String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters)); + throw new InvalidArgumentException( + String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters)); } } @@ -66,5 +67,4 @@ public class MetricCheckUtils { String sql = String.format("select %s from table", expr); return SqlSelectFunctionHelper.hasAggregateFunction(sql); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index e379401e8..23ed7322e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -70,11 +70,13 @@ public class MetricConverter { return convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList()); } - public static MetricResp convert2MetricResp(MetricDO metricDO, Map modelMap, List collect) { + public static MetricResp convert2MetricResp( + MetricDO metricDO, Map modelMap, List collect) { MetricResp metricResp = new MetricResp(); BeanUtils.copyProperties(metricDO, metricResp); - metricResp.setDataFormat(JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class)); + metricResp.setDataFormat( + JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class)); ModelResp modelResp = modelMap.get(metricDO.getModelId()); if (modelResp != null) { metricResp.setModelName(modelResp.getName()); @@ -88,21 +90,24 @@ public class MetricConverter { metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId())); metricResp.setClassifications(metricDO.getClassifications()); - metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(), - RelateDimension.class)); + metricResp.setRelateDimension( + JSONObject.parseObject(metricDO.getRelateDimensions(), RelateDimension.class)); if (metricDO.getExt() != null) { metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), HashMap.class)); } metricResp.setTypeEnum(TypeEnums.METRIC); if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) { - metricResp.setMetricDefineByMeasureParams(JSONObject.parseObject(metricDO.getTypeParams(), - MetricDefineByMeasureParams.class)); + metricResp.setMetricDefineByMeasureParams( + JSONObject.parseObject( + metricDO.getTypeParams(), MetricDefineByMeasureParams.class)); } else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) { - metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(), - MetricDefineByMetricParams.class)); + metricResp.setMetricDefineByMetricParams( + JSONObject.parseObject( + metricDO.getTypeParams(), MetricDefineByMetricParams.class)); } else if (MetricDefineType.FIELD.name().equalsIgnoreCase(metricDO.getDefineType())) { - metricResp.setMetricDefineByFieldParams(JSONObject.parseObject(metricDO.getTypeParams(), - MetricDefineByFieldParams.class)); + metricResp.setMetricDefineByFieldParams( + JSONObject.parseObject( + metricDO.getTypeParams(), MetricDefineByFieldParams.class)); } if (metricDO.getDefineType() != null) { metricResp.setMetricDefineType(MetricDefineType.valueOf(metricDO.getDefineType())); @@ -111,11 +116,15 @@ public class MetricConverter { return metricResp; } - public static List filterByDataSet(List metricResps, DataSetResp dataSetResp) { - return metricResps.stream().filter(metricResp -> - dataSetResp.metricIds().contains(metricResp.getId()) - || dataSetResp.getAllIncludeAllModels().contains(metricResp.getModelId())) + public static List filterByDataSet( + List metricResps, DataSetResp dataSetResp) { + return metricResps.stream() + .filter( + metricResp -> + dataSetResp.metricIds().contains(metricResp.getId()) + || dataSetResp + .getAllIncludeAllModels() + .contains(metricResp.getModelId())) .collect(Collectors.toList()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java index d9730fcc6..1c93afd6f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java @@ -27,8 +27,7 @@ import java.util.stream.Collectors; @Slf4j public class MetricDrillDownChecker { - @Autowired - private MetricService metricService; + @Autowired private MetricService metricService; public void checkQuery(QueryStatement queryStatement) { SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); @@ -49,11 +48,18 @@ public class MetricDrillDownChecker { } for (String metricName : metricFields) { MetricSchemaResp metric = semanticSchemaResp.getMetric(metricName); - List necessaryDimensions = getNecessaryDimensions(metric, semanticSchemaResp); - List dimensionsMissing = getNecessaryDimensionMissing(necessaryDimensions, dimensionFields); + List necessaryDimensions = + getNecessaryDimensions(metric, semanticSchemaResp); + List dimensionsMissing = + getNecessaryDimensionMissing(necessaryDimensions, dimensionFields); if (!CollectionUtils.isEmpty(dimensionsMissing)) { - String errMsg = String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(), - dimensionsMissing.stream().map(DimensionResp::getName).collect(Collectors.toList())); + String errMsg = + String.format( + "指标:%s 缺失必要下钻维度:%s", + metric.getName(), + dimensionsMissing.stream() + .map(DimensionResp::getName) + .collect(Collectors.toList())); throw new InvalidArgumentException(errMsg); } } @@ -67,51 +73,60 @@ public class MetricDrillDownChecker { if (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime()) { continue; } - String errMsg = String.format("维度:%s, 不在当前查询指标的下钻维度配置中, 请检查", dimSchemaResp.getName()); + String errMsg = + String.format("维度:%s, 不在当前查询指标的下钻维度配置中, 请检查", dimSchemaResp.getName()); throw new InvalidArgumentException(errMsg); } } } /** - * To check whether the dimension bound to the metric exists, - * eg: metric like UV is calculated in a certain dimension, it cannot be used on other dimensions. + * To check whether the dimension bound to the metric exists, eg: metric like UV is calculated + * in a certain dimension, it cannot be used on other dimensions. */ - private List getNecessaryDimensionMissing(List necessaryDimensions, - List dimensionFields) { + private List getNecessaryDimensionMissing( + List necessaryDimensions, List dimensionFields) { return necessaryDimensions.stream() .filter(dimension -> !dimensionFields.contains(dimension.getBizName())) .collect(Collectors.toList()); } /** - * To check whether the dimension can drill down the metric, - * eg: some descriptive dimensions are not suitable as drill-down dimensions + * To check whether the dimension can drill down the metric, eg: some descriptive dimensions are + * not suitable as drill-down dimensions */ - private boolean checkDrillDownDimension(String dimensionName, - List metricResps, - SemanticSchemaResp semanticSchemaResp) { + private boolean checkDrillDownDimension( + String dimensionName, + List metricResps, + SemanticSchemaResp semanticSchemaResp) { if (CollectionUtils.isEmpty(metricResps)) { return true; } - List relateDimensions = metricResps.stream() - .map(this::getDrillDownDimensions) - .filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions)) - .map(drillDownDimensions -> drillDownDimensions.stream() - .map(DrillDownDimension::getDimensionId).collect(Collectors.toList())) - .flatMap(Collection::stream) - .map(id -> convertDimensionIdToBizName(id, semanticSchemaResp)) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - //if no metric has drill down dimension, return true + List relateDimensions = + metricResps.stream() + .map(this::getDrillDownDimensions) + .filter( + drillDownDimensions -> + !CollectionUtils.isEmpty(drillDownDimensions)) + .map( + drillDownDimensions -> + drillDownDimensions.stream() + .map(DrillDownDimension::getDimensionId) + .collect(Collectors.toList())) + .flatMap(Collection::stream) + .map(id -> convertDimensionIdToBizName(id, semanticSchemaResp)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + // if no metric has drill down dimension, return true if (CollectionUtils.isEmpty(relateDimensions)) { return true; } - //if this dimension not in relate drill-down dimensions, return false + // if this dimension not in relate drill-down dimensions, return false return relateDimensions.contains(dimensionName); } - private List getNecessaryDimensions(MetricSchemaResp metric, SemanticSchemaResp semanticSchemaResp) { + private List getNecessaryDimensions( + MetricSchemaResp metric, SemanticSchemaResp semanticSchemaResp) { if (metric == null) { return Lists.newArrayList(); } @@ -120,7 +135,8 @@ public class MetricDrillDownChecker { return Lists.newArrayList(); } return drillDownDimensions.stream() - .filter(DrillDownDimension::isNecessary).map(DrillDownDimension::getDimensionId) + .filter(DrillDownDimension::isNecessary) + .map(DrillDownDimension::getDimensionId) .map(semanticSchemaResp::getDimension) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -137,7 +153,8 @@ public class MetricDrillDownChecker { return dimensionFields; } - private List getMetrics(List metricFields, SemanticSchemaResp semanticSchemaResp) { + private List getMetrics( + List metricFields, SemanticSchemaResp semanticSchemaResp) { return semanticSchemaResp.getMetrics().stream() .filter(metricSchemaResp -> metricFields.contains(metricSchemaResp.getBizName())) .collect(Collectors.toList()); @@ -154,5 +171,4 @@ public class MetricDrillDownChecker { private List getDrillDownDimensions(MetricResp metricResp) { return metricService.getDrillDownDimension(metricResp.getId()); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java index a808cd856..ba6e17e20 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.utils; - import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.server.pojo.ModelCluster; @@ -20,8 +19,11 @@ public class ModelClusterBuilder { public static Map buildModelClusters(List modelIds) { SchemaService schemaService = ContextUtils.getBean(SchemaService.class); List modelSchemaResps = schemaService.fetchModelSchemaResps(modelIds); - Map modelIdToModelSchema = modelSchemaResps.stream() - .collect(Collectors.toMap(ModelSchemaResp::getId, value -> value, (k1, k2) -> k1)); + Map modelIdToModelSchema = + modelSchemaResps.stream() + .collect( + Collectors.toMap( + ModelSchemaResp::getId, value -> value, (k1, k2) -> k1)); Set visited = new HashSet<>(); List> modelClusters = new ArrayList<>(); @@ -38,17 +40,25 @@ public class ModelClusterBuilder { .collect(Collectors.toMap(ModelCluster::getKey, value -> value, (k1, k2) -> k1)); } - private static ModelCluster getModelCluster(Map modelIdToModelSchema, Set modelIds) { - boolean containsPartitionDimensions = modelIds.stream() - .map(modelIdToModelSchema::get) - .filter(Objects::nonNull) - .anyMatch(modelSchemaResp -> CollectionUtils.isNotEmpty(modelSchemaResp.getTimeDimension())); + private static ModelCluster getModelCluster( + Map modelIdToModelSchema, Set modelIds) { + boolean containsPartitionDimensions = + modelIds.stream() + .map(modelIdToModelSchema::get) + .filter(Objects::nonNull) + .anyMatch( + modelSchemaResp -> + CollectionUtils.isNotEmpty( + modelSchemaResp.getTimeDimension())); return ModelCluster.build(modelIds, containsPartitionDimensions); } - private static void dfs(ModelSchemaResp model, Map modelMap, - Set visited, Set modelCluster) { + private static void dfs( + ModelSchemaResp model, + Map modelMap, + Set visited, + Set modelCluster) { visited.add(model.getId()); modelCluster.add(model.getId()); for (Long neighborId : model.getModelClusterSet()) { @@ -57,5 +67,4 @@ public class ModelClusterBuilder { } } } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index 3308e1606..1cf1e1894 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -36,7 +36,6 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; - public class ModelConverter { public static ModelDO convert(ModelReq modelReq, User user) { @@ -56,15 +55,24 @@ public class ModelConverter { public static ModelResp convert(ModelDO modelDO) { ModelResp modelResp = new ModelResp(); BeanUtils.copyProperties(modelDO, modelResp); - modelResp.setAdmins(StringUtils.isBlank(modelDO.getAdmin()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getAdmin().split(","))); - modelResp.setAdminOrgs(StringUtils.isBlank(modelDO.getAdminOrg()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getAdminOrg().split(","))); - modelResp.setViewers(StringUtils.isBlank(modelDO.getViewer()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getViewer().split(","))); - modelResp.setViewOrgs(StringUtils.isBlank(modelDO.getViewOrg()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getViewOrg().split(","))); - modelResp.setDrillDownDimensions(JsonUtil.toList(modelDO.getDrillDownDimensions(), DrillDownDimension.class)); + modelResp.setAdmins( + StringUtils.isBlank(modelDO.getAdmin()) + ? Lists.newArrayList() + : Arrays.asList(modelDO.getAdmin().split(","))); + modelResp.setAdminOrgs( + StringUtils.isBlank(modelDO.getAdminOrg()) + ? Lists.newArrayList() + : Arrays.asList(modelDO.getAdminOrg().split(","))); + modelResp.setViewers( + StringUtils.isBlank(modelDO.getViewer()) + ? Lists.newArrayList() + : Arrays.asList(modelDO.getViewer().split(","))); + modelResp.setViewOrgs( + StringUtils.isBlank(modelDO.getViewOrg()) + ? Lists.newArrayList() + : Arrays.asList(modelDO.getViewOrg().split(","))); + modelResp.setDrillDownDimensions( + JsonUtil.toList(modelDO.getDrillDownDimensions(), DrillDownDimension.class)); modelResp.setModelDetail(JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class)); modelResp.setExt(JsonUtil.toObject(modelDO.getExt(), Map.class)); return modelResp; @@ -83,7 +91,8 @@ public class ModelConverter { ModelDetail modelDetail = updateModelDetail(modelReq); BeanMapper.mapper(modelReq, modelDO); if (modelReq.getDrillDownDimensions() != null) { - modelDO.setDrillDownDimensions(JSONObject.toJSONString(modelReq.getDrillDownDimensions())); + modelDO.setDrillDownDimensions( + JSONObject.toJSONString(modelReq.getDrillDownDimensions())); } modelDO.setModelDetail(JSONObject.toJSONString((modelDetail))); if (modelReq.getExt() != null) { @@ -120,7 +129,8 @@ public class ModelConverter { dimensionReq.setModelId(modelDO.getId()); dimensionReq.setExpr(dim.getBizName()); dimensionReq.setType(dim.getType()); - dimensionReq.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); + dimensionReq.setDescription( + Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); dimensionReq.setIsTag(dim.getIsTag()); return dimensionReq; } @@ -156,24 +166,22 @@ public class ModelConverter { public static List convertList(List modelDOS) { List modelDescs = Lists.newArrayList(); if (!CollectionUtils.isEmpty(modelDOS)) { - modelDescs = modelDOS.stream().map(ModelConverter::convert).collect(Collectors.toList()); + modelDescs = + modelDOS.stream().map(ModelConverter::convert).collect(Collectors.toList()); } return modelDescs; } private static boolean isCreateDimension(Dim dim) { - return dim.getIsCreateDimension() == 1 - && StringUtils.isNotBlank(dim.getName()); + return dim.getIsCreateDimension() == 1 && StringUtils.isNotBlank(dim.getName()); } private static boolean isCreateDimension(Identify identify) { - return identify.getIsCreateDimension() == 1 - && StringUtils.isNotBlank(identify.getName()); + return identify.getIsCreateDimension() == 1 && StringUtils.isNotBlank(identify.getName()); } private static boolean isCreateMetric(Measure measure) { - return measure.getIsCreateMetric() == 1 - && StringUtils.isNotBlank(measure.getName()); + return measure.getIsCreateMetric() == 1 && StringUtils.isNotBlank(measure.getName()); } public static List getDimToCreateDimension(ModelDetail modelDetail) { @@ -205,30 +213,37 @@ public class ModelConverter { public static List convertDimensionList(ModelDO modelDO) { List dimensionReqs = Lists.newArrayList(); - ModelDetail modelDetail = JSONObject.parseObject(modelDO.getModelDetail(), - ModelDetail.class); + ModelDetail modelDetail = + JSONObject.parseObject(modelDO.getModelDetail(), ModelDetail.class); List dims = getDimToCreateDimension(modelDetail); if (!CollectionUtils.isEmpty(dims)) { - dimensionReqs = dims.stream().filter(dim -> StringUtils.isNotBlank(dim.getName())) - .map(dim -> convert(dim, modelDO)).collect(Collectors.toList()); + dimensionReqs = + dims.stream() + .filter(dim -> StringUtils.isNotBlank(dim.getName())) + .map(dim -> convert(dim, modelDO)) + .collect(Collectors.toList()); } List identifies = getIdentityToCreateDimension(modelDetail); if (CollectionUtils.isEmpty(identifies)) { return dimensionReqs; } - dimensionReqs.addAll(identifies.stream() - .map(identify -> convert(identify, modelDO)).collect(Collectors.toList())); + dimensionReqs.addAll( + identifies.stream() + .map(identify -> convert(identify, modelDO)) + .collect(Collectors.toList())); return dimensionReqs; } public static List convertMetricList(ModelDO modelDO) { - ModelDetail modelDetail = JSONObject.parseObject(modelDO.getModelDetail(), - ModelDetail.class); + ModelDetail modelDetail = + JSONObject.parseObject(modelDO.getModelDetail(), ModelDetail.class); List measures = getMeasureToCreateMetric(modelDetail); if (CollectionUtils.isEmpty(measures)) { return Lists.newArrayList(); } - return measures.stream().map(measure -> convert(measure, modelDO)).collect(Collectors.toList()); + return measures.stream() + .map(measure -> convert(measure, modelDO)) + .collect(Collectors.toList()); } private static ModelDetail createModelDetail(ModelReq modelReq) { @@ -258,9 +273,9 @@ public class ModelConverter { if (StringUtils.isBlank(measure.getBizName())) { continue; } - //Compatible with front-end tmp - String oriFieldName = measure.getBizName() - .replaceFirst(modelReq.getBizName() + "_", ""); + // Compatible with front-end tmp + String oriFieldName = + measure.getBizName().replaceFirst(modelReq.getBizName() + "_", ""); measure.setExpr(oriFieldName); if (!measure.getBizName().startsWith(modelReq.getBizName())) { measure.setBizName(String.format("%s_%s", modelReq.getBizName(), oriFieldName)); @@ -269,5 +284,4 @@ public class ModelConverter { BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); return modelDetail; } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/NameCheckUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/NameCheckUtils.java index 7171dd780..6c5830f6d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/NameCheckUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/NameCheckUtils.java @@ -32,5 +32,4 @@ public class NameCheckUtils { Matcher matcher = pattern.matcher(str); return matcher.find(); } - -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 2e47adc34..9b3f01369 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.utils; - import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; @@ -29,7 +28,6 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import java.util.Comparator; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -39,6 +37,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -52,23 +51,21 @@ import java.util.stream.Stream; @Slf4j public class QueryReqConverter { - @Autowired - private QueryStructUtils queryStructUtils; + @Autowired private QueryStructUtils queryStructUtils; - @Autowired - private SqlGenerateUtils sqlGenerateUtils; + @Autowired private SqlGenerateUtils sqlGenerateUtils; - public QueryStatement convert(QuerySqlReq querySQLReq, - SemanticSchemaResp semanticSchemaResp) throws Exception { + public QueryStatement convert(QuerySqlReq querySQLReq, SemanticSchemaResp semanticSchemaResp) + throws Exception { if (semanticSchemaResp == null) { return new QueryStatement(); } - //1.convert name to bizName + // 1.convert name to bizName convertNameToBizName(querySQLReq, semanticSchemaResp); - //2.functionName corrector + // 2.functionName corrector functionNameCorrector(querySQLReq, semanticSchemaResp); - //3.correct tableName + // 3.correct tableName correctTableName(querySQLReq); String tableName = SqlSelectHelper.getTableName(querySQLReq.getSql()); @@ -79,10 +76,11 @@ public class QueryReqConverter { String reqSql = querySQLReq.getSql(); querySQLReq.setSql(SqlReplaceHelper.replaceAggAliasOrderItem(querySQLReq.getSql())); log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, querySQLReq.getSql()); - //4.build MetricTables + // 4.build MetricTables List allFields = SqlSelectHelper.getAllSelectFields(querySQLReq.getSql()); List metricSchemas = getMetrics(semanticSchemaResp, allFields); - List metrics = metricSchemas.stream().map(m -> m.getBizName()).collect(Collectors.toList()); + List metrics = + metricSchemas.stream().map(m -> m.getBizName()).collect(Collectors.toList()); QueryStructReq queryStructReq = new QueryStructReq(); MetricTable metricTable = new MetricTable(); metricTable.setMetrics(metrics); @@ -95,31 +93,35 @@ public class QueryReqConverter { // if metric empty , fill model default if (CollectionUtils.isEmpty(metricTable.getMetrics())) { metricTable.setMetrics(new ArrayList<>()); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - semanticSchemaResp.getModelResps().get(0).getBizName())); + metricTable + .getMetrics() + .add( + sqlGenerateUtils.generateInternalMetricName( + semanticSchemaResp.getModelResps().get(0).getBizName())); } else { queryStructReq.setAggregators( - metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( - Collectors.toList())); + metricTable.getMetrics().stream() + .map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)) + .collect(Collectors.toList())); } AggOption aggOption = getAggOption(querySQLReq, metricSchemas); metricTable.setAggOption(aggOption); List tables = new ArrayList<>(); tables.add(metricTable); - //4.build ParseSqlReq + // 4.build ParseSqlReq DataSetQueryParam result = new DataSetQueryParam(); BeanUtils.copyProperties(querySQLReq, result); result.setTables(tables); DatabaseResp database = semanticSchemaResp.getDatabaseResp(); - if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), - database.getVersion())) { + if (!sqlGenerateUtils.isSupportWith( + EngineType.fromString(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); result.setWithAlias(false); } - //5. do deriveMetric + // 5. do deriveMetric generateDerivedMetric(semanticSchemaResp, aggOption, result); - //6.physicalSql by ParseSqlReq + // 6.physicalSql by ParseSqlReq queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); queryStructReq.setDataSetId(querySQLReq.getDataSetId()); @@ -157,11 +159,18 @@ public class QueryReqConverter { if (databaseReq.isInnerLayerNative()) { return AggOption.NATIVE; } - if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) { + if (SqlSelectHelper.hasSubSelect(sql) + || SqlSelectHelper.hasWith(sql) + || SqlSelectHelper.hasGroupBy(sql)) { return AggOption.OUTER; } - long defaultAggNullCnt = metricSchemas.stream() - .filter(m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg())).count(); + long defaultAggNullCnt = + metricSchemas.stream() + .filter( + m -> + Objects.isNull(m.getDefaultAgg()) + || StringUtils.isBlank(m.getDefaultAgg())) + .count(); if (defaultAggNullCnt > 0) { log.debug("getAggOption find null defaultAgg metric set to NATIVE"); return AggOption.OUTER; @@ -169,35 +178,54 @@ public class QueryReqConverter { return AggOption.DEFAULT; } - private void convertNameToBizName(QuerySqlReq querySqlReq, SemanticSchemaResp semanticSchemaResp) { + private void convertNameToBizName( + QuerySqlReq querySqlReq, SemanticSchemaResp semanticSchemaResp) { Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); String sql = querySqlReq.getSql(); - log.debug("dataSetId:{},convert name to bizName before:{}", querySqlReq.getDataSetId(), sql); + log.debug( + "dataSetId:{},convert name to bizName before:{}", querySqlReq.getDataSetId(), sql); String replaceFields = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); - log.debug("dataSetId:{},convert name to bizName after:{}", querySqlReq.getDataSetId(), replaceFields); + log.debug( + "dataSetId:{},convert name to bizName after:{}", + querySqlReq.getDataSetId(), + replaceFields); querySqlReq.setSql(replaceFields); } - private Set getDimensions(SemanticSchemaResp semanticSchemaResp, List allFields) { - Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() - .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName, - (k1, k2) -> k1)); - Map internalLowerToNameMap = QueryStructUtils.internalCols.stream() - .collect(Collectors.toMap(String::toLowerCase, a -> a)); + private Set getDimensions( + SemanticSchemaResp semanticSchemaResp, List allFields) { + Map dimensionLowerToNameMap = + semanticSchemaResp.getDimensions().stream() + .collect( + Collectors.toMap( + entry -> entry.getBizName().toLowerCase(), + SchemaItem::getBizName, + (k1, k2) -> k1)); + Map internalLowerToNameMap = + QueryStructUtils.internalCols.stream() + .collect(Collectors.toMap(String::toLowerCase, a -> a)); dimensionLowerToNameMap.putAll(internalLowerToNameMap); return allFields.stream() .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) - .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet()); + .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())) + .collect(Collectors.toSet()); } - private List getMetrics(SemanticSchemaResp semanticSchemaResp, List allFields) { - Map metricLowerToNameMap = semanticSchemaResp.getMetrics().stream() - .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); - return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) - .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList()); + private List getMetrics( + SemanticSchemaResp semanticSchemaResp, List allFields) { + Map metricLowerToNameMap = + semanticSchemaResp.getMetrics().stream() + .collect( + Collectors.toMap( + entry -> entry.getBizName().toLowerCase(), entry -> entry)); + return allFields.stream() + .filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) + .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())) + .collect(Collectors.toList()); } - private void functionNameCorrector(QuerySqlReq databaseReq, SemanticSchemaResp semanticSchemaResp) { + private void functionNameCorrector( + QuerySqlReq databaseReq, SemanticSchemaResp semanticSchemaResp) { DatabaseResp database = semanticSchemaResp.getDatabaseResp(); if (Objects.isNull(database) || Objects.isNull(database.getType())) { return; @@ -205,20 +233,33 @@ public class QueryReqConverter { String type = database.getType(); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase()); if (Objects.nonNull(engineAdaptor)) { - String functionNameCorrector = engineAdaptor.functionNameCorrector(databaseReq.getSql()); + String functionNameCorrector = + engineAdaptor.functionNameCorrector(databaseReq.getSql()); databaseReq.setSql(functionNameCorrector); } } protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { // support fieldName and field alias to bizName - Map dimensionResults = semanticSchemaResp.getDimensions().stream() - .flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + Map dimensionResults = + semanticSchemaResp.getDimensions().stream() + .flatMap( + entry -> + getPairStream( + entry.getAlias(), + entry.getName(), + entry.getBizName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - Map metricResults = semanticSchemaResp.getMetrics().stream() - .flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + Map metricResults = + semanticSchemaResp.getMetrics().stream() + .flatMap( + entry -> + getPairStream( + entry.getAlias(), + entry.getName(), + entry.getBizName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); @@ -226,7 +267,8 @@ public class QueryReqConverter { return dimensionResults; } - private Stream> getPairStream(String aliasStr, String name, String bizName) { + private Stream> getPairStream( + String aliasStr, String name, String bizName) { Set> elements = new HashSet<>(); elements.add(Pair.of(name, bizName)); if (StringUtils.isNotBlank(aliasStr)) { @@ -240,8 +282,9 @@ public class QueryReqConverter { public void correctTableName(QuerySqlReq querySqlReq) { String sql = querySqlReq.getSql(); - sql = SqlReplaceHelper.replaceTable(sql, - Constants.TABLE_PREFIX + querySqlReq.getDataSetId()); + sql = + SqlReplaceHelper.replaceTable( + sql, Constants.TABLE_PREFIX + querySqlReq.getDataSetId()); log.debug("correctTableName after:{}", sql); querySqlReq.setSql(sql); } @@ -255,15 +298,21 @@ public class QueryReqConverter { return queryType; } - private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, + private void generateDerivedMetric( + SemanticSchemaResp semanticSchemaResp, + AggOption aggOption, DataSetQueryParam viewQueryParam) { String sql = viewQueryParam.getSql(); for (MetricTable metricTable : viewQueryParam.getTables()) { Set measures = new HashSet<>(); Map replaces = new HashMap<>(); - generateDerivedMetric(semanticSchemaResp, aggOption, metricTable.getMetrics(), + generateDerivedMetric( + semanticSchemaResp, + aggOption, + metricTable.getMetrics(), metricTable.getDimensions(), - measures, replaces); + measures, + replaces); if (!CollectionUtils.isEmpty(replaces)) { // metricTable sql use measures replace metric sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces); @@ -274,48 +323,72 @@ public class QueryReqConverter { } else { // empty measure , fill default metricTable.setMetrics(new ArrayList<>()); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); + metricTable + .getMetrics() + .add( + sqlGenerateUtils.generateInternalMetricName( + getDefaultModel( + semanticSchemaResp, + metricTable.getDimensions()))); } } } viewQueryParam.setSql(sql); } - private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, - List metrics, List dimensions, - Set measures, Map replaces) { + private void generateDerivedMetric( + SemanticSchemaResp semanticSchemaResp, + AggOption aggOption, + List metrics, + List dimensions, + Set measures, + Map replaces) { List metricResps = semanticSchemaResp.getMetrics(); List dimensionResps = semanticSchemaResp.getDimensions(); // check metrics has derived if (!metricResps.stream() - .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(), - m.getMetricDefineByMeasureParams()))) { + .anyMatch( + m -> + metrics.contains(m.getBizName()) + && MetricType.isDerived( + m.getMetricDefineType(), + m.getMetricDefineByMeasureParams()))) { return; } log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics); Set allFields = new HashSet<>(); Map allMeasures = new HashMap<>(); - semanticSchemaResp.getModelResps().forEach(modelResp -> { - allFields.addAll(modelResp.getFieldList()); - if (Objects.nonNull(modelResp.getModelDetail().getMeasures())) { - modelResp.getModelDetail().getMeasures().stream() - .forEach(mm -> allMeasures.put(mm.getBizName(), mm)); - } - }); + semanticSchemaResp + .getModelResps() + .forEach( + modelResp -> { + allFields.addAll(modelResp.getFieldList()); + if (Objects.nonNull(modelResp.getModelDetail().getMeasures())) { + modelResp.getModelDetail().getMeasures().stream() + .forEach(mm -> allMeasures.put(mm.getBizName(), mm)); + } + }); Set deriveDimension = new HashSet<>(); Set deriveMetric = new HashSet<>(); Set visitedMetric = new HashSet<>(); if (!CollectionUtils.isEmpty(metricResps)) { for (MetricResp metricResp : metricResps) { if (metrics.contains(metricResp.getBizName())) { - if (MetricType.isDerived(metricResp.getMetricDefineType(), + if (MetricType.isDerived( + metricResp.getMetricDefineType(), metricResp.getMetricDefineByMeasureParams())) { - String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures, - dimensionResps, - sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), aggOption, - visitedMetric, - deriveMetric, deriveDimension); + String expr = + sqlGenerateUtils.generateDerivedMetric( + metricResps, + allFields, + allMeasures, + dimensionResps, + sqlGenerateUtils.getExpr(metricResp), + metricResp.getMetricDefineType(), + aggOption, + visitedMetric, + deriveMetric, + deriveDimension); replaces.put(metricResp.getBizName(), expr); log.debug("derived metric {}->{}", metricResp.getBizName(), expr); } else { @@ -325,20 +398,27 @@ public class QueryReqConverter { } } measures.addAll(deriveMetric); - deriveDimension.stream().filter(d -> !dimensions.contains(d)).forEach(d -> dimensions.add(d)); + deriveDimension.stream() + .filter(d -> !dimensions.contains(d)) + .forEach(d -> dimensions.add(d)); } private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List dimensions) { if (!CollectionUtils.isEmpty(dimensions)) { Map modelMatchCnt = new HashMap<>(); for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { - modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions().stream() - .filter(d -> dimensions.contains(d.getBizName())).count()); + modelMatchCnt.put( + modelResp.getBizName(), + modelResp.getModelDetail().getDimensions().stream() + .filter(d -> dimensions.contains(d.getBizName())) + .count()); } - return modelMatchCnt.entrySet().stream().sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) - .map(m -> m.getKey()).findFirst().orElse(""); + return modelMatchCnt.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) + .map(m -> m.getKey()) + .findFirst() + .orElse(""); } return semanticSchemaResp.getModelResps().get(0).getBizName(); } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryRuleConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryRuleConverter.java index bdd90fc42..706ab6d5f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryRuleConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryRuleConverter.java @@ -21,8 +21,10 @@ public class QueryRuleConverter { BeanUtils.copyProperties(queryRuleReq, queryRuleDO); queryRuleDO.setRuleType(queryRuleReq.getRuleType().name()); queryRuleDO.setRule(JsonUtil.toString(queryRuleReq.getRule())); - queryRuleDO.setAction(Objects.isNull(queryRuleReq.getAction()) ? "" : - JsonUtil.toString(queryRuleReq.getAction())); + queryRuleDO.setAction( + Objects.isNull(queryRuleReq.getAction()) + ? "" + : JsonUtil.toString(queryRuleReq.getAction())); queryRuleDO.setExt(JsonUtil.toString(queryRuleReq.getExt())); return queryRuleDO; @@ -33,8 +35,10 @@ public class QueryRuleConverter { BeanUtils.copyProperties(queryRuleDO, queryRuleResp); queryRuleResp.setRuleType(QueryRuleType.valueOf(queryRuleDO.getRuleType())); queryRuleResp.setRule(JsonUtil.toObject(queryRuleDO.getRule(), RuleInfo.class)); - queryRuleResp.setAction(StringUtils.isEmpty(queryRuleDO.getAction()) ? new ActionInfo() : - JsonUtil.toObject(queryRuleDO.getAction(), ActionInfo.class)); + queryRuleResp.setAction( + StringUtils.isEmpty(queryRuleDO.getAction()) + ? new ActionInfo() + : JsonUtil.toObject(queryRuleDO.getAction(), ActionInfo.class)); queryRuleResp.setExt(JsonUtil.toMap(queryRuleDO.getExt(), String.class, String.class)); return queryRuleResp; @@ -42,7 +46,8 @@ public class QueryRuleConverter { public static List convert2RespList(List queryRules) { List queryRuleRespList = new ArrayList<>(); - queryRules.stream().forEach(queryRuleDO -> queryRuleRespList.add(convert2Resp(queryRuleDO))); + queryRules.stream() + .forEach(queryRuleDO -> queryRuleRespList.add(convert2Resp(queryRuleDO))); return queryRuleRespList; } -} \ No newline at end of file +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 4cd7be617..30443f465 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; @@ -18,7 +19,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -45,13 +45,12 @@ import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; import static com.tencent.supersonic.common.pojo.Constants.MONTH; import static com.tencent.supersonic.common.pojo.Constants.WEEK; - @Slf4j @Component public class QueryStructUtils { - public static Set internalTimeCols = new HashSet<>( - Arrays.asList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month")); + public static Set internalTimeCols = + new HashSet<>(Arrays.asList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month")); public static Set internalCols; static { @@ -64,8 +63,10 @@ public class QueryStructUtils { private final SchemaService schemaService; private String variablePrefix = "'${"; - public QueryStructUtils(DateModeUtils dateModeUtils, - SqlFilterUtils sqlFilterUtils, SchemaService schemaService) { + public QueryStructUtils( + DateModeUtils dateModeUtils, + SqlFilterUtils sqlFilterUtils, + SchemaService schemaService) { this.dateModeUtils = dateModeUtils; this.sqlFilterUtils = sqlFilterUtils; @@ -77,8 +78,8 @@ public class QueryStructUtils { MetaFilter metaFilter = new MetaFilter(); metaFilter.setDataSetId(queryStructReq.getDataSetId()); List dimensions = schemaService.getDimensions(metaFilter); - Map> pair = dimensions.stream() - .collect(Collectors.groupingBy(DimensionResp::getBizName)); + Map> pair = + dimensions.stream().collect(Collectors.groupingBy(DimensionResp::getBizName)); for (String group : queryStructReq.getGroups()) { if (pair.containsKey(group)) { dimensionIds.add(pair.get(group).get(0).getId()); @@ -99,7 +100,8 @@ public class QueryStructUtils { MetaFilter metaFilter = new MetaFilter(); metaFilter.setDataSetId(queryStructCmd.getDataSetId()); List metrics = schemaService.getMetrics(metaFilter); - Map> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName)); + Map> pair = + metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName)); for (Aggregator agg : queryStructCmd.getAggregators()) { if (pair.containsKey(agg.getColumn())) { metricIds.add(pair.get(agg.getColumn()).get(0).getId()); @@ -119,7 +121,8 @@ public class QueryStructUtils { queryStructReq.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn())); resNameEnSet.addAll(queryStructReq.getGroups()); queryStructReq.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn())); - sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col)); + sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()).stream() + .forEach(col -> resNameEnSet.add(col)); return resNameEnSet; } @@ -127,33 +130,42 @@ public class QueryStructUtils { return new HashSet<>(SqlSelectHelper.getAllSelectFields(querySqlReq.getSql())); } - public Set getBizNameFromSql(QuerySqlReq querySqlReq, - SemanticSchemaResp semanticSchemaResp) { + public Set getBizNameFromSql( + QuerySqlReq querySqlReq, SemanticSchemaResp semanticSchemaResp) { Set resNameSet = getResName(querySqlReq); Set resNameEnSet = new HashSet<>(); if (semanticSchemaResp != null) { List metrics = semanticSchemaResp.getMetrics(); List dimensions = semanticSchemaResp.getDimensions(); - metrics.stream().forEach(o -> { - if (resNameSet.contains(o.getName()) || resNameSet.contains(o.getBizName())) { - resNameEnSet.add(o.getBizName()); - } - }); - dimensions.stream().forEach(o -> { - if (resNameSet.contains(o.getName()) || resNameSet.contains(o.getBizName())) { - resNameEnSet.add(o.getBizName()); - } - }); + metrics.stream() + .forEach( + o -> { + if (resNameSet.contains(o.getName()) + || resNameSet.contains(o.getBizName())) { + resNameEnSet.add(o.getBizName()); + } + }); + dimensions.stream() + .forEach( + o -> { + if (resNameSet.contains(o.getName()) + || resNameSet.contains(o.getBizName())) { + resNameEnSet.add(o.getBizName()); + } + }); } - return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); + return resNameEnSet.stream() + .filter(res -> !internalCols.contains(res)) + .collect(Collectors.toSet()); } public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) { List dimensionIds = getDimensionIds(queryStructCmd); List metricIds = getMetricIds(queryStructCmd); - ItemDateResp dateDate = schemaService.getItemDate( - new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.name()), - new ItemDateFilter(metricIds, TypeEnums.METRIC.name())); + ItemDateResp dateDate = + schemaService.getItemDate( + new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.name()), + new ItemDateFilter(metricIds, TypeEnums.METRIC.name())); return dateDate; } @@ -171,28 +183,39 @@ public class QueryStructUtils { case BETWEEN: return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate()); case LIST: - return Triple.of(dateInfo, Collections.min(dateConf.getDateList()), + return Triple.of( + dateInfo, + Collections.min(dateConf.getDateList()), Collections.max(dateConf.getDateList())); case RECENT: ItemDateResp dateDate = getItemDateResp(queryStructCmd); LocalDate dateMax = LocalDate.now().minusDays(1); LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1); if (Objects.isNull(dateDate)) { - return Triple.of(dateInfo, dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)), + return Triple.of( + dateInfo, + dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)), dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT))); } switch (dateConf.getPeriod()) { case DAY: - ImmutablePair dayInfo = dateModeUtils.recentDay(dateDate, dateConf); + ImmutablePair dayInfo = + dateModeUtils.recentDay(dateDate, dateConf); return Triple.of(dateInfo, dayInfo.left, dayInfo.right); case WEEK: - ImmutablePair weekInfo = dateModeUtils.recentWeek(dateDate, dateConf); + ImmutablePair weekInfo = + dateModeUtils.recentWeek(dateDate, dateConf); return Triple.of(dateInfo, weekInfo.left, weekInfo.right); case MONTH: - List> rets = dateModeUtils.recentMonth(dateDate, dateConf); - Optional minBegins = rets.stream().map(i -> i.left).sorted().findFirst(); - Optional maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder()) - .findFirst(); + List> rets = + dateModeUtils.recentMonth(dateDate, dateConf); + Optional minBegins = + rets.stream().map(i -> i.left).sorted().findFirst(); + Optional maxBegins = + rets.stream() + .map(i -> i.right) + .sorted(Comparator.reverseOrder()) + .findFirst(); if (minBegins.isPresent() && maxBegins.isPresent()) { return Triple.of(dateInfo, minBegins.get(), maxBegins.get()); } @@ -203,7 +226,6 @@ public class QueryStructUtils { break; default: break; - } return Triple.of("", "", ""); } @@ -216,10 +238,12 @@ public class QueryStructUtils { String endDate = ""; String period = ""; for (FieldExpression f : fieldExpressions) { - if (Objects.isNull(f.getFieldName()) || !internalCols.contains(f.getFieldName().toLowerCase())) { + if (Objects.isNull(f.getFieldName()) + || !internalCols.contains(f.getFieldName().toLowerCase())) { continue; } - if (Objects.isNull(f.getFieldValue()) || !dateModeUtils.isDateStr(f.getFieldValue().toString())) { + if (Objects.isNull(f.getFieldValue()) + || !dateModeUtils.isDateStr(f.getFieldValue().toString())) { continue; } period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase()); @@ -229,7 +253,8 @@ public class QueryStructUtils { if ("=".equals(f.getOperator())) { dateList.add(f.getFieldValue().toString()); } else if ("<".equals(f.getOperator()) || "<=".equals(f.getOperator())) { - if (startDate.isEmpty() || startDate.compareTo(f.getFieldValue().toString()) > 0) { + if (startDate.isEmpty() + || startDate.compareTo(f.getFieldValue().toString()) > 0) { startDate = f.getFieldValue().toString(); } } else if (">".equals(f.getOperator()) || ">=".equals(f.getOperator())) { @@ -256,6 +281,4 @@ public class QueryStructUtils { } return null; } - } - diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 6f0f85479..d2631faf3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -26,7 +26,6 @@ import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; - @Slf4j @Component public class QueryUtils { @@ -38,7 +37,8 @@ public class QueryUtils { @Value("${s2.query-optimizer.enable:true}") private Boolean optimizeEnable; - public void populateQueryColumns(SemanticQueryResp semanticQueryResp, SemanticSchemaResp semanticSchemaResp) { + public void populateQueryColumns( + SemanticQueryResp semanticQueryResp, SemanticSchemaResp semanticSchemaResp) { Map metricRespMap = createMetricRespMap(semanticSchemaResp); Map namePair = new HashMap<>(); Map nameTypePair = new HashMap<>(); @@ -49,35 +49,45 @@ public class QueryUtils { private Map createMetricRespMap(SemanticSchemaResp semanticSchemaResp) { List metrics = semanticSchemaResp.getMetrics(); - return metrics.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); + return metrics.stream() + .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); } - private void populateNamePairs(SemanticSchemaResp semanticSchemaResp, - Map namePair, - Map nameTypePair) { + private void populateNamePairs( + SemanticSchemaResp semanticSchemaResp, + Map namePair, + Map nameTypePair) { for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) { namePair.put(timeDimensionEnum.getName(), "date"); nameTypePair.put(timeDimensionEnum.getName(), "DATE"); } - semanticSchemaResp.getMetrics().forEach(metricDesc -> { - namePair.put(metricDesc.getBizName(), metricDesc.getName()); - nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name()); - }); - semanticSchemaResp.getDimensions().forEach(dimensionDesc -> { - namePair.put(dimensionDesc.getBizName(), dimensionDesc.getName()); - nameTypePair.put(dimensionDesc.getBizName(), dimensionDesc.getSemanticType()); - }); + semanticSchemaResp + .getMetrics() + .forEach( + metricDesc -> { + namePair.put(metricDesc.getBizName(), metricDesc.getName()); + nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name()); + }); + semanticSchemaResp + .getDimensions() + .forEach( + dimensionDesc -> { + namePair.put(dimensionDesc.getBizName(), dimensionDesc.getName()); + nameTypePair.put( + dimensionDesc.getBizName(), dimensionDesc.getSemanticType()); + }); } - private void processColumn(QueryColumn column, - Map namePair, - Map nameTypePair, - Map metricRespMap) { + private void processColumn( + QueryColumn column, + Map namePair, + Map nameTypePair, + Map metricRespMap) { String nameEn = getName(column.getNameEn().toLowerCase()); if (nameEn.contains(JOIN_UNDERLINE)) { nameEn = nameEn.split(JOIN_UNDERLINE)[1]; } - //set name + // set name if (namePair.containsKey(nameEn)) { column.setName(namePair.get(nameEn)); } else { @@ -85,12 +95,13 @@ public class QueryUtils { if (StringUtils.isEmpty(nameEnByRegex)) { nameEnByRegex = getNameEnByRegex(nameEn, no_quotation_pattern); } - if (StringUtils.isNotEmpty(nameEnByRegex) && StringUtils.isNotEmpty(namePair.get(nameEnByRegex))) { + if (StringUtils.isNotEmpty(nameEnByRegex) + && StringUtils.isNotEmpty(namePair.get(nameEnByRegex))) { String filedName = namePair.get(nameEnByRegex); column.setName(nameEn.replaceAll(nameEnByRegex, filedName)); } } - //set showType + // set showType if (nameTypePair.containsKey(nameEn)) { column.setShowType(nameTypePair.get(nameEn)); } @@ -100,7 +111,7 @@ public class QueryUtils { if (StringUtils.isEmpty(column.getShowType())) { column.setShowType(SemanticType.CATEGORY.name()); } - //set dataFormat/dataFormatType + // set dataFormat/dataFormatType if (metricRespMap.containsKey(nameEn)) { column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType()); column.setDataFormat(metricRespMap.get(nameEn).getDataFormat()); @@ -111,10 +122,13 @@ public class QueryUtils { if (StringUtils.isBlank(type)) { return false; } - return type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint") - || type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double") + return type.equalsIgnoreCase("int") + || type.equalsIgnoreCase("bigint") + || type.equalsIgnoreCase("float") + || type.equalsIgnoreCase("double") || type.equalsIgnoreCase("numeric") - || type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int"); + || type.toLowerCase().startsWith("uint") + || type.toLowerCase().startsWith("int"); } private String getName(String nameEn) { @@ -137,17 +151,23 @@ public class QueryUtils { return null; } - public QueryStatement sqlParserUnion(QueryMultiStructReq queryMultiStructCmd, List sqlParsers) { + public QueryStatement sqlParserUnion( + QueryMultiStructReq queryMultiStructCmd, List sqlParsers) { QueryStatement sqlParser = new QueryStatement(); StringBuilder unionSqlBuilder = new StringBuilder(); for (int i = 0; i < sqlParsers.size(); i++) { - String selectStr = SqlGenerateUtils.getUnionSelect(queryMultiStructCmd.getQueryStructReqs().get(i)); - unionSqlBuilder.append(String.format("select %s from ( %s ) sub_sql_%s", - selectStr, - sqlParsers.get(i).getSql(), i)); + String selectStr = + SqlGenerateUtils.getUnionSelect( + queryMultiStructCmd.getQueryStructReqs().get(i)); + unionSqlBuilder.append( + String.format( + "select %s from ( %s ) sub_sql_%s", + selectStr, sqlParsers.get(i).getSql(), i)); unionSqlBuilder.append(UNIONALL); } - String unionSql = unionSqlBuilder.substring(0, unionSqlBuilder.length() - Constants.UNIONALL.length()); + String unionSql = + unionSqlBuilder.substring( + 0, unionSqlBuilder.length() - Constants.UNIONALL.length()); sqlParser.setSql(unionSql); log.info("union sql parser:{}", sqlParser); return sqlParser; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index b248a8ec0..9496aa71c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -33,20 +33,17 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -; - - @Component @Slf4j public class StatUtils { - private static final TransmittableThreadLocal STATS = new TransmittableThreadLocal<>(); + private static final TransmittableThreadLocal STATS = + new TransmittableThreadLocal<>(); private final StatRepository statRepository; private final SqlFilterUtils sqlFilterUtils; private final ObjectMapper objectMapper = new ObjectMapper(); - public StatUtils(StatRepository statRepository, - SqlFilterUtils sqlFilterUtils) { + public StatUtils(StatRepository statRepository, SqlFilterUtils sqlFilterUtils) { this.statRepository = statRepository; this.sqlFilterUtils = sqlFilterUtils; @@ -68,12 +65,15 @@ public class StatUtils { QueryStat queryStatInfo = get(); queryStatInfo.setElapsedMs(System.currentTimeMillis() - queryStatInfo.getStartTime()); queryStatInfo.setQueryState(state.getStatus()); - CompletableFuture.runAsync(() -> { - statRepository.createRecord(queryStatInfo); - }).exceptionally(exception -> { - log.warn("queryStatInfo, exception:", exception); - return null; - }); + CompletableFuture.runAsync( + () -> { + statRepository.createRecord(queryStatInfo); + }) + .exceptionally( + exception -> { + log.warn("queryStatInfo, exception:", exception); + return null; + }); remove(); } @@ -91,7 +91,8 @@ public class StatUtils { initStructStatInfo((QueryStructReq) semanticQueryReq, facadeUser); } if (semanticQueryReq instanceof QueryMultiStructReq) { - QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); + QueryStructReq queryStructCmd = + ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); initStructStatInfo(queryStructCmd, facadeUser); } if (semanticQueryReq instanceof QueryTagReq) { @@ -105,11 +106,13 @@ public class StatUtils { List dimensions = queryTagReq.getGroups(); List metrics = new ArrayList<>(); - queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn())); + queryTagReq.getAggregators().stream() + .forEach(aggregator -> metrics.add(aggregator.getColumn())); String user = getUserName(facadeUser); try { - queryStatInfo.setTraceId(traceId) + queryStatInfo + .setTraceId(traceId) .setDataSetId(queryTagReq.getDataSetId()) .setUser(user) .setQueryType(QueryMethod.STRUCT.getValue()) @@ -121,8 +124,9 @@ public class StatUtils { .setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups())) .setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators())) .setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders())) - .setFilterCols(objectMapper.writeValueAsString( - sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters()))) + .setFilterCols( + objectMapper.writeValueAsString( + sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters()))) .setUseResultCache(true) .setUseSqlCache(true) .setMetrics(objectMapper.writeValueAsString(metrics)) @@ -135,18 +139,19 @@ public class StatUtils { log.error("", e); } StatUtils.set(queryStatInfo); - } public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); List aggFields = SqlSelectHelper.getAggregateFields(querySqlReq.getSql()); List allFields = SqlSelectHelper.getAllSelectFields(querySqlReq.getSql()); - List dimensions = allFields.stream().filter(aggFields::contains).collect(Collectors.toList()); + List dimensions = + allFields.stream().filter(aggFields::contains).collect(Collectors.toList()); String userName = getUserName(facadeUser); try { - queryStatInfo.setTraceId("") + queryStatInfo + .setTraceId("") .setUser(userName) .setDataSetId(querySqlReq.getDataSetId()) .setQueryType(QueryMethod.SQL.getValue()) @@ -173,11 +178,13 @@ public class StatUtils { List dimensions = queryStructReq.getGroups(); List metrics = new ArrayList<>(); - queryStructReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn())); + queryStructReq.getAggregators().stream() + .forEach(aggregator -> metrics.add(aggregator.getColumn())); String user = getUserName(facadeUser); try { - queryStatInfo.setTraceId(traceId) + queryStatInfo + .setTraceId(traceId) .setDataSetId(queryStructReq.getDataSetId()) .setUser(user) .setQueryType(QueryMethod.STRUCT.getValue()) @@ -189,8 +196,10 @@ public class StatUtils { .setGroupByCols(objectMapper.writeValueAsString(queryStructReq.getGroups())) .setAggCols(objectMapper.writeValueAsString(queryStructReq.getAggregators())) .setOrderByCols(objectMapper.writeValueAsString(queryStructReq.getOrders())) - .setFilterCols(objectMapper.writeValueAsString( - sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()))) + .setFilterCols( + objectMapper.writeValueAsString( + sqlFilterUtils.getFiltersCol( + queryStructReq.getOriginalFilter()))) .setUseResultCache(true) .setUseSqlCache(true) .setMetrics(objectMapper.writeValueAsString(metrics)) @@ -203,23 +212,25 @@ public class StatUtils { log.error("", e); } StatUtils.set(queryStatInfo); - } - private List getFieldNames(List allFields, List schemaItems) { - Set fieldNames = schemaItems - .stream() - .map(dimSchemaResp -> dimSchemaResp.getBizName()) - .collect(Collectors.toSet()); + private List getFieldNames( + List allFields, List schemaItems) { + Set fieldNames = + schemaItems.stream() + .map(dimSchemaResp -> dimSchemaResp.getBizName()) + .collect(Collectors.toSet()); if (!CollectionUtils.isEmpty(fieldNames)) { - return allFields.stream().filter(fieldName -> fieldNames.contains(fieldName)) + return allFields.stream() + .filter(fieldName -> fieldNames.contains(fieldName)) .collect(Collectors.toList()); } return new ArrayList<>(); } private String getUserName(User facadeUser) { - return (Objects.nonNull(facadeUser) && StringUtils.isNotEmpty(facadeUser.getName())) ? facadeUser.getName() + return (Objects.nonNull(facadeUser) && StringUtils.isNotEmpty(facadeUser.getName())) + ? facadeUser.getName() : "Admin"; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagObjectConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagObjectConverter.java index 632cca930..5aba1724b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagObjectConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagObjectConverter.java @@ -33,8 +33,9 @@ public class TagObjectConverter { public static List convert2RespList(List tagObjectDOList) { List tagObjectRespList = new ArrayList<>(); if (!CollectionUtils.isEmpty(tagObjectDOList)) { - tagObjectDOList.stream().forEach(tagObjectDO -> tagObjectRespList.add(convert2Resp(tagObjectDO))); + tagObjectDOList.stream() + .forEach(tagObjectDO -> tagObjectRespList.add(convert2Resp(tagObjectDO))); } return tagObjectRespList; } -} \ No newline at end of file +} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownCheckerTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownCheckerTest.java index e3bcb1386..0ec5da101 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownCheckerTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownCheckerTest.java @@ -10,9 +10,10 @@ import com.tencent.supersonic.headless.server.utils.DataUtils; import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertThrows; @Slf4j public class MetricDrillDownCheckerTest { @@ -30,7 +31,8 @@ public class MetricDrillDownCheckerTest { MetricDrillDownChecker metricDrillDownChecker = new MetricDrillDownChecker(); String sql = "select page, sum(pv) from t_1 group by page"; SemanticSchemaResp semanticSchemaResp = mockModelSchemaResp(); - assertThrows(InvalidArgumentException.class, + assertThrows( + InvalidArgumentException.class, () -> metricDrillDownChecker.checkQuery(semanticSchemaResp, sql)); } @@ -39,7 +41,8 @@ public class MetricDrillDownCheckerTest { MetricDrillDownChecker metricDrillDownChecker = new MetricDrillDownChecker(); String sql = "select user_name, count(distinct uv) from t_1 group by user_name"; SemanticSchemaResp semanticSchemaResp = mockModelSchemaResp(); - assertThrows(InvalidArgumentException.class, + assertThrows( + InvalidArgumentException.class, () -> metricDrillDownChecker.checkQuery(semanticSchemaResp, sql)); } @@ -68,24 +71,29 @@ public class MetricDrillDownCheckerTest { private SemanticSchemaResp mockModelSchemaNoDimensionSetting() { SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp(); - List metricSchemaResps = Lists.newArrayList(mockMetricsNoDrillDownSetting()); + List metricSchemaResps = + Lists.newArrayList(mockMetricsNoDrillDownSetting()); semanticSchemaResp.setMetrics(metricSchemaResps); semanticSchemaResp.setDimensions(mockDimensions()); return semanticSchemaResp; } private List mockDimensions() { - return Lists.newArrayList(DataUtils.mockDimension(1L, "user_name", "用户名"), + return Lists.newArrayList( + DataUtils.mockDimension(1L, "user_name", "用户名"), DataUtils.mockDimension(2L, "department", "部门"), DataUtils.mockDimension(3L, "page", "页面")); } private List mockMetrics() { return Lists.newArrayList( - DataUtils.mockMetric(1L, "pv", "访问次数", + DataUtils.mockMetric( + 1L, + "pv", + "访问次数", Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(2L))), - DataUtils.mockMetric(2L, "uv", "访问用户数", - Lists.newArrayList(new DrillDownDimension(2L, true)))); + DataUtils.mockMetric( + 2L, "uv", "访问用户数", Lists.newArrayList(new DrillDownDimension(2L, true)))); } private List mockMetricsNoDrillDownSetting() { @@ -93,5 +101,4 @@ public class MetricDrillDownCheckerTest { DataUtils.mockMetric(1L, "pv", Lists.newArrayList()), DataUtils.mockMetric(2L, "uv", Lists.newArrayList())); } - } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index 42576cec9..cc1897086 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -4,10 +4,10 @@ import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp; -import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; +import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; @@ -16,20 +16,21 @@ import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.extern.slf4j.Slf4j; @Slf4j class HeadlessParserServiceTest { private static Map headlessSchemaMap = new HashMap<>(); - public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryParam metricQueryParam, - boolean isAgg) { + public static SqlParserResp parser( + SemanticSchema semanticSchema, MetricQueryParam metricQueryParam, boolean isAgg) { SqlParserResp sqlParser = new SqlParserResp(); try { if (semanticSchema == null) { @@ -40,7 +41,9 @@ class HeadlessParserServiceTest { QueryStatement queryStatement = new QueryStatement(); queryStatement.setMetricQueryParam(metricQueryParam); aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); - EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); + EngineType engineType = + EngineType.fromString( + semanticSchema.getSemanticModel().getDatabase().getType()); sqlParser.setSql(aggBuilder.getSql(engineType)); sqlParser.setSourceId(aggBuilder.getSourceId()); } catch (Exception e) { @@ -55,7 +58,8 @@ class HeadlessParserServiceTest { DataModelYamlTpl datasource = new DataModelYamlTpl(); datasource.setName("s2_pv_uv_statis"); datasource.setSourceId(1L); - datasource.setSqlQuery("SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis"); + datasource.setSqlQuery( + "SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis"); MeasureYamlTpl measure = new MeasureYamlTpl(); measure.setAgg("sum"); @@ -121,7 +125,8 @@ class HeadlessParserServiceTest { datasource.setIdentifiers(identifies); SemanticSchema semanticSchema = SemanticSchema.newBuilder("1").build(); - SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource)); + SemanticSchemaManager.update( + semanticSchema, SemanticSchemaManager.getDatasource(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("page"); @@ -130,7 +135,9 @@ class HeadlessParserServiceTest { List dimensionYamlTpls = new ArrayList<>(); dimensionYamlTpls.add(dimension1); - SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis", + SemanticSchemaManager.update( + semanticSchema, + "s2_pv_uv_statis", SemanticSchemaManager.getDimensions(dimensionYamlTpls)); MetricYamlTpl metric1 = new MetricYamlTpl(); @@ -160,12 +167,13 @@ class HeadlessParserServiceTest { metric2.setTypeParams(metricTypeParams1); metric.add(metric2); - //HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); + // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); MetricQueryParam metricCommand = new MetricQueryParam(); metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date"))); metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv"))); - metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); + metricCommand.setWhere( + "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); metricCommand.setLimit(1000L); List orders = new ArrayList<>(); orders.add(ColumnOrder.buildDesc("sys_imp_date")); @@ -175,8 +183,13 @@ class HeadlessParserServiceTest { addDepartment(semanticSchema); MetricQueryParam metricCommand2 = new MetricQueryParam(); - metricCommand2.setDimensions(new ArrayList<>( - Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page"))); + metricCommand2.setDimensions( + new ArrayList<>( + Arrays.asList( + "sys_imp_date", + "user_name__department", + "user_name", + "user_name__page"))); metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand2.setWhere( "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); @@ -185,8 +198,6 @@ class HeadlessParserServiceTest { orders2.add(ColumnOrder.buildDesc("sys_imp_date")); metricCommand2.setOrder(orders2); System.out.println(parser(semanticSchema, metricCommand2, true)); - - } private static void addDepartment(SemanticSchema semanticSchema) { @@ -235,7 +246,9 @@ class HeadlessParserServiceTest { identifies.add(identify); datasource.setIdentifiers(identifies); - semanticSchema.getDatasource().put("user_department", SemanticSchemaManager.getDatasource(datasource)); + semanticSchema + .getDatasource() + .put("user_department", SemanticSchemaManager.getDatasource(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("department"); @@ -244,7 +257,8 @@ class HeadlessParserServiceTest { List dimensionYamlTpls = new ArrayList<>(); dimensionYamlTpls.add(dimension1); - semanticSchema.getDimension() + semanticSchema + .getDimension() .put("user_department", SemanticSchemaManager.getDimensions(dimensionYamlTpls)); } } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java index 969b140ec..e494ca516 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java @@ -16,12 +16,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - class DownloadServiceImplTest { @Test - void testBatchDownload() throws Exception { - - } + void testBatchDownload() throws Exception {} private ModelSchemaResp mockModelSchemaResp() { ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); @@ -38,14 +35,17 @@ class DownloadServiceImplTest { return modelSchemaResp; } - private MetricSchemaResp mockMetric(Long id, String bizName, String name, List drillDownloadDimensions) { + private MetricSchemaResp mockMetric( + Long id, String bizName, String name, List drillDownloadDimensions) { MetricSchemaResp metricResp = new MetricSchemaResp(); metricResp.setId(id); metricResp.setBizName(bizName); metricResp.setName(name); RelateDimension relateDimension = new RelateDimension(); - relateDimension.setDrillDownDimensions(drillDownloadDimensions.stream() - .map(DrillDownDimension::new).collect(Collectors.toList())); + relateDimension.setDrillDownDimensions( + drillDownloadDimensions.stream() + .map(DrillDownDimension::new) + .collect(Collectors.toList())); metricResp.setRelateDimension(relateDimension); return metricResp; } @@ -93,7 +93,8 @@ class DownloadServiceImplTest { return semanticQueryResp; } - private static Map createMap(String sysImpDate, String d1, String d2, String m1) { + private static Map createMap( + String sysImpDate, String d1, String d2, String m1) { Map map = new HashMap<>(); map.put("sys_imp_date", sysImpDate); map.put("user_name", d1); @@ -101,5 +102,4 @@ class DownloadServiceImplTest { map.put("pv", m1); return map; } - } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index a684cc996..811442935 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -19,10 +19,10 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; -import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; -import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.service.impl.DataSetServiceImpl; import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl; +import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; +import com.tencent.supersonic.headless.server.utils.MetricConverter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -63,8 +63,8 @@ public class MetricServiceImplTest { Assertions.assertEquals(expectedMetricResp, actualMetricResp); } - private MetricService mockMetricService(MetricRepository metricRepository, - ModelService modelService) { + private MetricService mockMetricService( + MetricRepository metricRepository, ModelService modelService) { AliasGenerateHelper aliasGenerateHelper = Mockito.mock(AliasGenerateHelper.class); CollectService collectService = Mockito.mock(CollectService.class); ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class); @@ -72,9 +72,16 @@ public class MetricServiceImplTest { DimensionService dimensionService = Mockito.mock(DimensionService.class); TagMetaService tagMetaService = Mockito.mock(TagMetaService.class); ChatLayerService chatLayerService = Mockito.mock(ChatLayerService.class); - return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper, - collectService, dataSetService, eventPublisher, dimensionService, - tagMetaService, chatLayerService); + return new MetricServiceImpl( + metricRepository, + modelService, + aliasGenerateHelper, + collectService, + dataSetService, + eventPublisher, + dimensionService, + tagMetaService, + chatLayerService); } private MetricReq buildMetricReq() { @@ -92,17 +99,20 @@ public class MetricServiceImplTest { dataFormat.setNeedMultiply100(false); metricReq.setDataFormat(dataFormat); MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams(); - typeParams.setMeasures(Lists.newArrayList( - new MeasureParam("s2_pv", "department='hr'"), - new MeasureParam("s2_uv", "department='hr'"))); + typeParams.setMeasures( + Lists.newArrayList( + new MeasureParam("s2_pv", "department='hr'"), + new MeasureParam("s2_uv", "department='hr'"))); typeParams.setExpr("s2_pv/s2_uv"); metricReq.setMetricDefineByMeasureParams(typeParams); metricReq.setClassifications(Lists.newArrayList("核心指标")); metricReq.setRelateDimension( - RelateDimension.builder().drillDownDimensions(Lists.newArrayList( - new DrillDownDimension(1L), - new DrillDownDimension(1L, false)) - ).build()); + RelateDimension.builder() + .drillDownDimensions( + Lists.newArrayList( + new DrillDownDimension(1L), + new DrillDownDimension(1L, false))) + .build()); metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode()); metricReq.setExt(new HashMap<>()); return metricReq; @@ -123,17 +133,20 @@ public class MetricServiceImplTest { dataFormat.setNeedMultiply100(false); metricResp.setDataFormat(dataFormat); MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams(); - typeParams.setMeasures(Lists.newArrayList( - new MeasureParam("s2_pv", "department='hr'"), - new MeasureParam("s2_uv", "department='hr'"))); + typeParams.setMeasures( + Lists.newArrayList( + new MeasureParam("s2_pv", "department='hr'"), + new MeasureParam("s2_uv", "department='hr'"))); typeParams.setExpr("s2_pv/s2_uv"); metricResp.setMetricDefineByMeasureParams(typeParams); metricResp.setClassifications("核心指标"); metricResp.setRelateDimension( - RelateDimension.builder().drillDownDimensions(Lists.newArrayList( - new DrillDownDimension(1L), - new DrillDownDimension(1L, false)) - ).build()); + RelateDimension.builder() + .drillDownDimensions( + Lists.newArrayList( + new DrillDownDimension(1L), + new DrillDownDimension(1L, false))) + .build()); metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode()); metricResp.setExt(new HashMap<>()); metricResp.setTypeEnum(TypeEnums.METRIC); @@ -150,9 +163,10 @@ public class MetricServiceImplTest { metricReq.setBizName("pv"); metricReq.setMetricDefineType(MetricDefineType.MEASURE); MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams(); - typeParams.setMeasures(Lists.newArrayList( - new MeasureParam("s2_pv", "department='hr'"), - new MeasureParam("s2_uv", "department='hr'"))); + typeParams.setMeasures( + Lists.newArrayList( + new MeasureParam("s2_pv", "department='hr'"), + new MeasureParam("s2_uv", "department='hr'"))); typeParams.setExpr("s2_pv/s2_uv"); metricReq.setMetricDefineByMeasureParams(typeParams); return metricReq; @@ -164,5 +178,4 @@ public class MetricServiceImplTest { modelResp.setDomainId(1L); return modelResp; } - } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java index 219987564..7c6211dc2 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java @@ -5,13 +5,13 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; -import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; +import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; @@ -34,8 +34,7 @@ class ModelServiceImplTest { void createModel() throws Exception { ModelRepository modelRepository = Mockito.mock(ModelRepository.class); ModelService modelService = mockModelService(modelRepository); - ModelResp actualModelResp = modelService.createModel( - mockModelReq(), User.getFakeUser()); + ModelResp actualModelResp = modelService.createModel(mockModelReq(), User.getFakeUser()); ModelResp expectedModelResp = buildExpectedModelResp(); Assertions.assertEquals(expectedModelResp, actualModelResp); } @@ -76,9 +75,15 @@ class ModelServiceImplTest { UserService userService = Mockito.mock(UserService.class); DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); DataSetService viewService = Mockito.mock(DataSetService.class); - return new ModelServiceImpl(modelRepository, databaseService, - dimensionService, metricService, domainService, userService, - viewService, dateInfoRepository); + return new ModelServiceImpl( + modelRepository, + databaseService, + dimensionService, + metricService, + domainService, + userService, + viewService, + dateInfoRepository); } private ModelReq mockModelReq() { @@ -109,7 +114,8 @@ class ModelServiceImplTest { Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); measures.add(measure2); modelDetail.setMeasures(measures); - modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); + modelDetail.setSqlQuery( + "SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelReq.setDomainId(1L); modelReq.setFilterSql("where user_name = 'alice'"); @@ -150,8 +156,9 @@ class ModelServiceImplTest { measures.add(measure2); modelDetail.setMeasures(measures); - modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a," - + " user_name as uv_a FROM s2_pv_uv_statis"); + modelDetail.setSqlQuery( + "SELECT imp_date_a, user_name_a, page_a, 1 as pv_a," + + " user_name as uv_a FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelReq.setDomainId(1L); modelReq.setFilterSql("where user_name = 'tom'"); @@ -182,8 +189,9 @@ class ModelServiceImplTest { Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); measures.add(measure2); modelDetail.setMeasures(measures); - modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, " - + "user_name as uv FROM s2_pv_uv_statis"); + modelDetail.setSqlQuery( + "SELECT imp_date, user_name, page, 1 as pv, " + + "user_name as uv FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelReq.setModelDetail(modelDetail); return modelReq; @@ -220,12 +228,14 @@ class ModelServiceImplTest { measure1.setExpr("pv"); measures.add(measure1); - Measure measure2 = new Measure("访问人数", "s2_pv_uv_statis_uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); + Measure measure2 = + new Measure("访问人数", "s2_pv_uv_statis_uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); measure2.setExpr("uv"); measures.add(measure2); modelDetail.setMeasures(measures); - modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); + modelDetail.setSqlQuery( + "SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelResp.setModelDetail(modelDetail); modelResp.setId(1L); @@ -259,24 +269,28 @@ class ModelServiceImplTest { modelDetail.setDimensions(dimensions); List measures = new ArrayList<>(); - Measure measure1 = new Measure("访问次数_a", "s2_pv_uv_statis_a_pv_a", - AggOperatorEnum.SUM.name(), 1); + Measure measure1 = + new Measure("访问次数_a", "s2_pv_uv_statis_a_pv_a", AggOperatorEnum.SUM.name(), 1); measure1.setExpr("pv_a"); measures.add(measure1); - Measure measure2 = new Measure("访问人数_a", "s2_pv_uv_statis_a_uv_a", - AggOperatorEnum.COUNT_DISTINCT.name(), 1); + Measure measure2 = + new Measure( + "访问人数_a", + "s2_pv_uv_statis_a_uv_a", + AggOperatorEnum.COUNT_DISTINCT.name(), + 1); measure2.setExpr("uv_a"); measures.add(measure2); modelDetail.setMeasures(measures); - modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, " - + "user_name as uv_a FROM s2_pv_uv_statis"); + modelDetail.setSqlQuery( + "SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, " + + "user_name as uv_a FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelResp.setModelDetail(modelDetail); modelResp.setId(1L); modelResp.setFilterSql("where user_name = 'tom'"); return modelResp; } - } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelperTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelperTest.java index 0e3afe9c7..64a66f4ce 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelperTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelperTest.java @@ -1,4 +1,5 @@ package com.tencent.supersonic.headless.server.utils; + import org.junit.jupiter.api.Test; class AliasGenerateHelperTest { @@ -6,13 +7,7 @@ class AliasGenerateHelperTest { @Test void extractJsonStringFromAiMessage1() { - /** - * { - * "name": "Alice", - * "age": 25, - * "city": "New York" - * } - */ + /** { "name": "Alice", "age": 25, "city": "New York" } */ String testJson1 = "{\"name\": \"Alice\", \"age\": 25, \"city\": \"New York\"}"; AliasGenerateHelper.extractJsonStringFromAiMessage(testJson1); } @@ -20,68 +15,54 @@ class AliasGenerateHelperTest { @Test void extractJsonStringFromAiMessage2() { - /** - * ``` - * { - * "name": "Alice", - * "age": 25, - * "city": "New York" - * } - * ``` - */ - String testJson2 = "```\n" - + "{\n" - + " \"name\": \"Alice\",\n" - + " \"age\": 25,\n" - + " \"city\": \"New York\"\n" - + "}\n" - + "```"; + /** ``` { "name": "Alice", "age": 25, "city": "New York" } ``` */ + String testJson2 = + "```\n" + + "{\n" + + " \"name\": \"Alice\",\n" + + " \"age\": 25,\n" + + " \"city\": \"New York\"\n" + + "}\n" + + "```"; AliasGenerateHelper.extractJsonStringFromAiMessage(testJson2); - } @Test void extractJsonStringFromAiMessage3() { /** - * I understand that you want me to generate a JSON object with two properties: `tran` and `alias`.... - * ```json - * { - * "name": "Alice", - * "age": 25, - * "city": "New York" - * } - * ``` - * Please let me know if there is any problem. + * I understand that you want me to generate a JSON object with two properties: `tran` and + * `alias`.... ```json { "name": "Alice", "age": 25, "city": "New York" } ``` Please let me + * know if there is any problem. */ - String testJson3 = "I understand that you want me to generate a JSON object with two properties: " - + "`tran` and `alias`...." - + "```json\n" - + "{\n" - + " \"name\": \"Alice\",\n" - + " \"age\": 25,\n" - + " \"city\": \"New York\"\n" - + "}\n" - + "```" - + "Please let me know if there is any problem."; + String testJson3 = + "I understand that you want me to generate a JSON object with two properties: " + + "`tran` and `alias`...." + + "```json\n" + + "{\n" + + " \"name\": \"Alice\",\n" + + " \"age\": 25,\n" + + " \"city\": \"New York\"\n" + + "}\n" + + "```" + + "Please let me know if there is any problem."; AliasGenerateHelper.extractJsonStringFromAiMessage(testJson3); - } @Test void extractJsonStringFromAiMessage4() { - String testJson4 = "Based on the provided JSON-schema, I will construct the answer as follows:\n" - + "\n" - + "[\n" - + " \"作者名称\",\n" - + " \"作者姓名\",\n" - + " \"创作者\",\n" - + " \"作者信息\"\n" - + "]\n" - + "\n" - + "This answer conforms to the format described in the JSON-schema"; + String testJson4 = + "Based on the provided JSON-schema, I will construct the answer as follows:\n" + + "\n" + + "[\n" + + " \"作者名称\",\n" + + " \"作者姓名\",\n" + + " \"创作者\",\n" + + " \"作者信息\"\n" + + "]\n" + + "\n" + + "This answer conforms to the format described in the JSON-schema"; AliasGenerateHelper.extractJsonStringFromAiMessage(testJson4); - } -} \ No newline at end of file +} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java index dfef206f9..96dba41f1 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java @@ -23,12 +23,13 @@ class DataTransformUtilsTest { inputData.add(createMap("2023/10/15", "b", "b", "5")); List groups = Lists.newArrayList("d1", "d2"); String metric = "m1"; - List> resultData = DataTransformUtils.transform(inputData, - metric, groups, new DateConf()); + List> resultData = + DataTransformUtils.transform(inputData, metric, groups, new DateConf()); Assertions.assertEquals(3, resultData.size()); } - private static Map createMap(String sysImpDate, String d1, String d2, String m1) { + private static Map createMap( + String sysImpDate, String d1, String d2, String m1) { Map map = new HashMap<>(); map.put("sys_imp_date", sysImpDate); map.put("d1", d1); @@ -36,5 +37,4 @@ class DataTransformUtilsTest { map.put("m1", m1); return map; } - } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataUtils.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataUtils.java index a81496bc9..c56c63bd2 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataUtils.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataUtils.java @@ -26,20 +26,19 @@ public class DataUtils { return metricSchemaResp; } - public static MetricSchemaResp mockMetric(Long id, String bizName, String name, - List drillDownDimensions) { + public static MetricSchemaResp mockMetric( + Long id, String bizName, String name, List drillDownDimensions) { MetricSchemaResp metricSchemaResp = new MetricSchemaResp(); metricSchemaResp.setId(id); metricSchemaResp.setName(name); metricSchemaResp.setBizName(bizName); - metricSchemaResp.setRelateDimension(RelateDimension.builder() - .drillDownDimensions(drillDownDimensions).build()); + metricSchemaResp.setRelateDimension( + RelateDimension.builder().drillDownDimensions(drillDownDimensions).build()); return metricSchemaResp; } - public static MetricSchemaResp mockMetric(Long id, String bizName, - List drillDownDimensions) { + public static MetricSchemaResp mockMetric( + Long id, String bizName, List drillDownDimensions) { return mockMetric(id, bizName, null, drillDownDimensions); } - } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/NameCheckUtilsTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/NameCheckUtilsTest.java index 2b21c6ca2..d81b1af7c 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/NameCheckUtilsTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/NameCheckUtilsTest.java @@ -9,7 +9,8 @@ class NameCheckUtilsTest { @Test void findForbiddenCharacters() { Assert.assertTrue(StringUtils.isBlank(NameCheckUtils.findForbiddenCharacters("访问时长"))); - Assert.assertTrue(StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长(秒)"))); + Assert.assertTrue( + StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长(秒)"))); Assert.assertTrue(StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长#"))); Assert.assertTrue(StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长%"))); Assert.assertTrue(StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长("))); @@ -17,4 +18,4 @@ class NameCheckUtilsTest { Assert.assertTrue(StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长（"))); Assert.assertTrue(StringUtils.isNotBlank(NameCheckUtils.findForbiddenCharacters("访问时长）"))); } -} \ No newline at end of file +} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java index cb6cb5b8e..9c1394130 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.server.utils; - import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; @@ -21,9 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -/** - * QueryNLReqBuilderTest - */ +/** QueryNLReqBuilderTest */ class QueryNLReqBuilderTest { @Test @@ -56,7 +53,8 @@ class QueryNLReqBuilderTest { Assert.assertEquals( "SELECT department, SUM(pv) AS pv FROM 内容库 " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " - + "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); + + "BY department ORDER BY uv LIMIT 2000", + querySQLReq.getSql()); queryStructReq.setQueryType(QueryType.DETAIL); querySQLReq = queryStructReq.convert(); @@ -64,17 +62,20 @@ class QueryNLReqBuilderTest { "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "ORDER BY uv LIMIT 2000", querySQLReq.getSql()); - } private void init() { MockedStatic mockContextUtils = Mockito.mockStatic(ContextUtils.class); SqlFilterUtils sqlFilterUtils = new SqlFilterUtils(); - mockContextUtils.when(() -> ContextUtils.getBean(SqlFilterUtils.class)).thenReturn(sqlFilterUtils); + mockContextUtils + .when(() -> ContextUtils.getBean(SqlFilterUtils.class)) + .thenReturn(sqlFilterUtils); DateModeUtils dateModeUtils = new DateModeUtils(); - mockContextUtils.when(() -> ContextUtils.getBean(DateModeUtils.class)).thenReturn(dateModeUtils); + mockContextUtils + .when(() -> ContextUtils.getBean(DateModeUtils.class)) + .thenReturn(dateModeUtils); dateModeUtils.setSysDateCol("sys_imp_date"); dateModeUtils.setSysDateWeekCol("sys_imp_week"); dateModeUtils.setSysDateMonthCol("sys_imp_month"); } -} \ No newline at end of file +} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java index c9e119082..07ea12e26 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.VariableValueType; import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; + import java.util.List; public class SqlVariableParseUtilsTest { @@ -14,8 +15,9 @@ public class SqlVariableParseUtilsTest { @Test void testParseSql_defaultVariableValue() { String sql = "select * from t_$interval$ where id = $id$ and name = $name$"; - List variables = Lists.newArrayList(mockNumSqlVariable(), - mockExprSqlVariable(), mockStrSqlVariable()); + List variables = + Lists.newArrayList( + mockNumSqlVariable(), mockExprSqlVariable(), mockStrSqlVariable()); String actualSql = SqlVariableParseUtils.parse(sql, variables, Lists.newArrayList()); String expectedSql = "select * from t_d where id = 1 and name = 'tom'"; Assertions.assertEquals(expectedSql, actualSql); @@ -24,9 +26,11 @@ public class SqlVariableParseUtilsTest { @Test void testParseSql() { String sql = "select * from t_$interval$ where id = $id$ and name = $name$"; - List variables = Lists.newArrayList(mockNumSqlVariable(), - mockExprSqlVariable(), mockStrSqlVariable()); - List params = Lists.newArrayList(mockIdParam(), mockNameParam(), mockIntervalParam()); + List variables = + Lists.newArrayList( + mockNumSqlVariable(), mockExprSqlVariable(), mockStrSqlVariable()); + List params = + Lists.newArrayList(mockIdParam(), mockNameParam(), mockIntervalParam()); String actualSql = SqlVariableParseUtils.parse(sql, variables, params); String expectedSql = "select * from t_wk where id = 2 and name = 'alice'"; Assertions.assertEquals(expectedSql, actualSql); @@ -44,7 +48,8 @@ public class SqlVariableParseUtilsTest { return mockSqlVariable("interval", VariableValueType.EXPR, "d"); } - private SqlVariable mockSqlVariable(String name, VariableValueType variableValueType, Object value) { + private SqlVariable mockSqlVariable( + String name, VariableValueType variableValueType, Object value) { SqlVariable sqlVariable = new SqlVariable(); sqlVariable.setName(name); sqlVariable.setValueType(variableValueType); @@ -67,5 +72,4 @@ public class SqlVariableParseUtilsTest { private Param mockParam(String name, String value) { return new Param(name, value); } - } diff --git a/launchers/chat/src/main/java/com/tencent/supersonic/ChatLauncher.java b/launchers/chat/src/main/java/com/tencent/supersonic/ChatLauncher.java index d33b82313..862fd7fbd 100644 --- a/launchers/chat/src/main/java/com/tencent/supersonic/ChatLauncher.java +++ b/launchers/chat/src/main/java/com/tencent/supersonic/ChatLauncher.java @@ -6,10 +6,9 @@ import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfigurat import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; import org.springframework.scheduling.annotation.EnableScheduling; -/** - * Chat Launcher - **/ -@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic"}, +/** Chat Launcher */ +@SpringBootApplication( + scanBasePackages = {"com.tencent.supersonic"}, exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) @EnableScheduling public class ChatLauncher { diff --git a/launchers/chat/src/main/java/com/tencent/supersonic/db/MybatisConfig.java b/launchers/chat/src/main/java/com/tencent/supersonic/db/MybatisConfig.java index 7856043bd..859720814 100644 --- a/launchers/chat/src/main/java/com/tencent/supersonic/db/MybatisConfig.java +++ b/launchers/chat/src/main/java/com/tencent/supersonic/db/MybatisConfig.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.db; import javax.sql.DataSource; + import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.session.SqlSessionFactory; import org.mybatis.spring.SqlSessionFactoryBean; @@ -9,7 +10,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; - @Configuration @MapperScan(value = "com.tencent.supersonic", annotationClass = Mapper.class) public class MybatisConfig { @@ -19,12 +19,14 @@ public class MybatisConfig { @Bean public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception { SqlSessionFactoryBean bean = new SqlSessionFactoryBean(); - org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); + org.apache.ibatis.session.Configuration configuration = + new org.apache.ibatis.session.Configuration(); configuration.setMapUnderscoreToCamelCase(true); bean.setConfiguration(configuration); bean.setDataSource(dataSource); - bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(MAPPER_LOCATION)); + bean.setMapperLocations( + new PathMatchingResourcePatternResolver().getResources(MAPPER_LOCATION)); return bean.getObject(); } } diff --git a/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java b/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java index 928df91b1..103d50019 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java @@ -20,22 +20,28 @@ import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; @RestControllerAdvice(annotations = RestController.class) public class ResponseAdvice implements ResponseBodyAdvice { - @Autowired - private ObjectMapper objectMapper; + @Autowired private ObjectMapper objectMapper; @Override - public boolean supports(MethodParameter methodParameter, Class> aClass) { + public boolean supports( + MethodParameter methodParameter, Class> aClass) { return !methodParameter.getDeclaringClass().isAssignableFrom(BasicErrorController.class); } @SneakyThrows @Override - public Object beforeBodyWrite(Object result, MethodParameter methodParameter, MediaType mediaType, - Class> aClass, ServerHttpRequest serverHttpRequest, - ServerHttpResponse serverHttpResponse) { + public Object beforeBodyWrite( + Object result, + MethodParameter methodParameter, + MediaType mediaType, + Class> aClass, + ServerHttpRequest serverHttpRequest, + ServerHttpResponse serverHttpResponse) { // 判断当前请求是否是 Swagger 相关的请求 String path = serverHttpRequest.getURI().getPath(); - if (path.startsWith("/swagger") || path.startsWith("/v3/api-docs") || path.startsWith("/v2/api-docs")) { + if (path.startsWith("/swagger") + || path.startsWith("/v3/api-docs") + || path.startsWith("/v2/api-docs")) { return result; } objectMapper.registerModule(new JavaTimeModule()); @@ -47,4 +53,4 @@ public class ResponseAdvice implements ResponseBodyAdvice { } return ResultData.success(result); } -} \ No newline at end of file +} diff --git a/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java b/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java index cbb651a29..8d8209878 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.advice; +import com.tencent.supersonic.common.pojo.ResultData; +import com.tencent.supersonic.common.pojo.enums.ReturnCode; import com.tencent.supersonic.common.pojo.exception.AccessException; import com.tencent.supersonic.common.pojo.exception.CommonException; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; -import com.tencent.supersonic.common.pojo.ResultData; -import com.tencent.supersonic.common.pojo.enums.ReturnCode; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.ExceptionHandler; @@ -16,9 +16,7 @@ import org.springframework.web.bind.annotation.RestControllerAdvice; @RestControllerAdvice public class RestExceptionHandler { - /** - * default global exception handler - */ + /** default global exception handler */ @ExceptionHandler(Exception.class) @ResponseStatus(HttpStatus.OK) public ResultData exception(Exception e) { @@ -53,5 +51,4 @@ public class RestExceptionHandler { log.error("default global exception", e); return ResultData.fail(e.getCode(), e.getMessage()); } - -} \ No newline at end of file +} diff --git a/launchers/common/src/main/java/com/tencent/supersonic/config/RestTemplateConfig.java b/launchers/common/src/main/java/com/tencent/supersonic/config/RestTemplateConfig.java index 7bc0d3d6f..2623e91db 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/config/RestTemplateConfig.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/config/RestTemplateConfig.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.config; -import java.nio.charset.StandardCharsets; import org.apache.http.client.HttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.LaxRedirectStrategy; @@ -10,19 +9,25 @@ import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.web.client.RestTemplate; +import java.nio.charset.StandardCharsets; + @Configuration public class RestTemplateConfig { @Bean public RestTemplate restTemplate() { - HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory(); + HttpComponentsClientHttpRequestFactory httpRequestFactory = + new HttpComponentsClientHttpRequestFactory(); httpRequestFactory.setConnectionRequestTimeout(2000); httpRequestFactory.setConnectTimeout(10000); httpRequestFactory.setReadTimeout(7200000); - HttpClient httpClient = HttpClientBuilder.create().setRedirectStrategy(new LaxRedirectStrategy()).build(); + HttpClient httpClient = + HttpClientBuilder.create().setRedirectStrategy(new LaxRedirectStrategy()).build(); httpRequestFactory.setHttpClient(httpClient); RestTemplate restTemplate = new RestTemplate(httpRequestFactory); - restTemplate.getMessageConverters().set(1, new StringHttpMessageConverter(StandardCharsets.UTF_8)); + restTemplate + .getMessageConverters() + .set(1, new StringHttpMessageConverter(StandardCharsets.UTF_8)); return restTemplate; } } diff --git a/launchers/common/src/main/java/com/tencent/supersonic/config/SpringContextUtil.java b/launchers/common/src/main/java/com/tencent/supersonic/config/SpringContextUtil.java index 64e293132..4ee6654cc 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/config/SpringContextUtil.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/config/SpringContextUtil.java @@ -31,4 +31,4 @@ public class SpringContextUtil implements ApplicationContextAware { return "dev"; } -} \ No newline at end of file +} diff --git a/launchers/common/src/main/java/com/tencent/supersonic/web/ErrorPageConfig.java b/launchers/common/src/main/java/com/tencent/supersonic/web/ErrorPageConfig.java index 70710a8da..ed25ff189 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/web/ErrorPageConfig.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/web/ErrorPageConfig.java @@ -6,9 +6,7 @@ import org.springframework.boot.web.server.ErrorPageRegistry; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; -/** - * error page config - */ +/** error page config */ @Component public class ErrorPageConfig implements ErrorPageRegistrar { @@ -17,4 +15,4 @@ public class ErrorPageConfig implements ErrorPageRegistrar { ErrorPage error404Page = new ErrorPage(HttpStatus.NOT_FOUND, "/webapp/index.html"); registry.addErrorPages(error404Page); } -} \ No newline at end of file +} diff --git a/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java b/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java index 1718cd582..b6cf5fde1 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java @@ -10,12 +10,11 @@ public class WebConfig extends WebMvcConfigurerAdapter { @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { - registry.addResourceHandler("/webapp/**") - .addResourceLocations("classpath:/webapp/"); + registry.addResourceHandler("/webapp/**").addResourceLocations("classpath:/webapp/"); } @Override public void addViewControllers(ViewControllerRegistry registry) { registry.addViewController("/").setViewName("forward:webapp/index.html"); } -} \ No newline at end of file +} diff --git a/launchers/headless/src/main/java/com/tencent/supersonic/HeadlessLauncher.java b/launchers/headless/src/main/java/com/tencent/supersonic/HeadlessLauncher.java index f56320acb..f856c73cd 100644 --- a/launchers/headless/src/main/java/com/tencent/supersonic/HeadlessLauncher.java +++ b/launchers/headless/src/main/java/com/tencent/supersonic/HeadlessLauncher.java @@ -6,11 +6,10 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -/** - * Headless Launcher - **/ +/** Headless Launcher */ @Slf4j -@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic"}, +@SpringBootApplication( + scanBasePackages = {"com.tencent.supersonic"}, exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) public class HeadlessLauncher { diff --git a/launchers/headless/src/main/java/com/tencent/supersonic/db/MybatisConfig.java b/launchers/headless/src/main/java/com/tencent/supersonic/db/MybatisConfig.java index 970deaf61..d34cf9b34 100644 --- a/launchers/headless/src/main/java/com/tencent/supersonic/db/MybatisConfig.java +++ b/launchers/headless/src/main/java/com/tencent/supersonic/db/MybatisConfig.java @@ -1,7 +1,8 @@ package com.tencent.supersonic.db; -import com.github.pagehelper.PageInterceptor; import javax.sql.DataSource; + +import com.github.pagehelper.PageInterceptor; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.plugin.Interceptor; import org.apache.ibatis.session.SqlSessionFactory; @@ -11,7 +12,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; - @Configuration @MapperScan(value = "com.tencent.supersonic", annotationClass = Mapper.class) public class MybatisConfig { @@ -19,15 +19,17 @@ public class MybatisConfig { private static final String MAPPER_LOCATION = "classpath*:mapper/**/*.xml"; @Bean - public SqlSessionFactory sqlSessionFactory(DataSource dataSource, PageInterceptor pageInterceptor) - throws Exception { + public SqlSessionFactory sqlSessionFactory( + DataSource dataSource, PageInterceptor pageInterceptor) throws Exception { SqlSessionFactoryBean bean = new SqlSessionFactoryBean(); - org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); + org.apache.ibatis.session.Configuration configuration = + new org.apache.ibatis.session.Configuration(); configuration.setMapUnderscoreToCamelCase(true); bean.setConfiguration(configuration); bean.setDataSource(dataSource); - bean.setPlugins(new Interceptor[]{pageInterceptor}); - bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(MAPPER_LOCATION)); + bean.setPlugins(new Interceptor[] {pageInterceptor}); + bean.setMapperLocations( + new PathMatchingResourcePatternResolver().getResources(MAPPER_LOCATION)); return bean.getObject(); } -} \ No newline at end of file +} diff --git a/launchers/headless/src/main/java/com/tencent/supersonic/db/PageHelperConfig.java b/launchers/headless/src/main/java/com/tencent/supersonic/db/PageHelperConfig.java index 495078ca9..099389468 100644 --- a/launchers/headless/src/main/java/com/tencent/supersonic/db/PageHelperConfig.java +++ b/launchers/headless/src/main/java/com/tencent/supersonic/db/PageHelperConfig.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.db; import com.github.pagehelper.PageInterceptor; -import java.util.Properties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.Properties; @Configuration public class PageHelperConfig { diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/StandaloneLauncher.java b/launchers/standalone/src/main/java/com/tencent/supersonic/StandaloneLauncher.java index a6de486bd..ce57affe6 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/StandaloneLauncher.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/StandaloneLauncher.java @@ -8,7 +8,8 @@ import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; import springfox.documentation.swagger2.annotations.EnableSwagger2; -@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic", "dev.langchain4j"}, +@SpringBootApplication( + scanBasePackages = {"com.tencent.supersonic", "dev.langchain4j"}, exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) @EnableScheduling @EnableAsync diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/config/SwaggerConfiguration.java b/launchers/standalone/src/main/java/com/tencent/supersonic/config/SwaggerConfiguration.java index 228bcb5e3..901846c78 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/config/SwaggerConfiguration.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/config/SwaggerConfiguration.java @@ -22,56 +22,39 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2; @EnableOpenApi public class SwaggerConfiguration { - /** - * 标题 - */ + /** 标题 */ @Value("${swagger.title}") private String title; - /** - * 基本包 - */ + /** 基本包 */ @Value("${swagger.base.package}") private String basePackage; - /** - * 描述 - */ + /** 描述 */ @Value("${swagger.description}") private String description; - /** - * URL - */ + /** URL */ @Value("${swagger.url}") private String url; - /** - * 作者 - */ + /** 作者 */ @Value("${swagger.contact.name}") private String contactName; - /** - * 作者网址 - */ + /** 作者网址 */ @Value("${swagger.contact.url}") private String contactUrl; - /** - * 作者邮箱 - */ + /** 作者邮箱 */ @Value("${swagger.contact.email}") private String contactEmail; - /** - * 版本 - */ + /** 版本 */ @Value("${swagger.version}") private String version; - @Autowired - private AuthenticationConfig authenticationConfig; + @Autowired private AuthenticationConfig authenticationConfig; @Bean public Docket createRestApi() { @@ -81,13 +64,15 @@ public class SwaggerConfiguration { .select() .apis(RequestHandlerSelectors.basePackage(basePackage)) .paths(PathSelectors.any()) - .build().securitySchemes(Lists.newArrayList(apiKey())); - + .build() + .securitySchemes(Lists.newArrayList(apiKey())); } private ApiKey apiKey() { - return new ApiKey(authenticationConfig.getTokenHttpHeaderKey(), - authenticationConfig.getTokenHttpHeaderKey(), "header"); + return new ApiKey( + authenticationConfig.getTokenHttpHeaderKey(), + authenticationConfig.getTokenHttpHeaderKey(), + "header"); } private ApiInfo apiInfo() { @@ -99,4 +84,4 @@ public class SwaggerConfiguration { .version(version) .build(); } -} \ No newline at end of file +} diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java b/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java index 7362fb763..f36faf093 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.db; +import javax.sql.DataSource; + import com.baomidou.mybatisplus.core.MybatisConfiguration; import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import org.apache.ibatis.annotations.Mapper; @@ -9,9 +11,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -import javax.sql.DataSource; - - @Configuration @MapperScan(value = "com.tencent.supersonic", annotationClass = Mapper.class) public class MybatisConfig { @@ -26,7 +25,8 @@ public class MybatisConfig { bean.setConfiguration(configuration); bean.setDataSource(dataSource); - bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(MAPPER_LOCATION)); + bean.setMapperLocations( + new PathMatchingResourcePatternResolver().getResources(MAPPER_LOCATION)); return bean.getObject(); } } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java index 2c272ae25..6853b80b1 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java @@ -8,23 +8,23 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import com.tencent.supersonic.headless.api.pojo.DefaultDisplayInfo; -import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.Dim; -import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.DataSetDetail; import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; +import com.tencent.supersonic.headless.api.pojo.DefaultDisplayInfo; +import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; +import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; @@ -53,7 +53,7 @@ public class CspiderDemo extends S2BaseDemo { addModelRela_3(s2Domain, songModelResp, artistModelResp); addModelRela_4(s2Domain, songModelResp, genreModelResp); addModelRela_5(s2Domain, songModelResp, filesModelResp); - //batchPushlishMetric(); + // batchPushlishMetric(); } catch (Exception e) { log.error("Failed to add bench mark demo data", e); } @@ -149,7 +149,7 @@ public class CspiderDemo extends S2BaseDemo { List dimensions = new ArrayList<>(); dimensions.add(new Dim("持续时间", "duration", DimensionType.categorical.name(), 1)); dimensions.add(new Dim("文件格式", "formats", DimensionType.categorical.name(), 1)); - //dimensions.add(new Dim("艺术家名称", "artist_name", DimensionType.categorical.name(), 1)); + // dimensions.add(new Dim("艺术家名称", "artist_name", DimensionType.categorical.name(), 1)); modelDetail.setDimensions(dimensions); List identifiers = new ArrayList<>(); @@ -160,7 +160,8 @@ public class CspiderDemo extends S2BaseDemo { modelDetail.setMeasures(Collections.emptyList()); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files"); + modelDetail.setSqlQuery( + "SELECT f_id, artist_name, file_size, duration, formats FROM files"); modelReq.setModelDetail(modelDetail); return modelService.createModel(modelReq, user); } @@ -187,7 +188,7 @@ public class CspiderDemo extends S2BaseDemo { identifiers.add(new Identify("歌曲名称", IdentifyType.primary.name(), "song_name")); identifiers.add(new Identify("歌曲ID", IdentifyType.foreign.name(), "f_id")); identifiers.add(new Identify("艺术家名称", IdentifyType.foreign.name(), "artist_name")); - //identifiers.add(new Identify("艺术家名称", IdentifyType.foreign.name(), "artist_name")); + // identifiers.add(new Identify("艺术家名称", IdentifyType.foreign.name(), "artist_name")); modelDetail.setIdentifiers(identifiers); @@ -197,8 +198,9 @@ public class CspiderDemo extends S2BaseDemo { modelDetail.setMeasures(measures); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, " - + " rating, languages, releasedate, resolution FROM song"); + modelDetail.setSqlQuery( + "SELECT imp_date, song_name, artist_name, country, f_id, g_name, " + + " rating, languages, releasedate, resolution FROM song"); modelReq.setModelDetail(modelDetail); return modelService.createModel(modelReq, user); } @@ -236,7 +238,8 @@ public class CspiderDemo extends S2BaseDemo { dataSetService.save(dataSetReq, User.getFakeUser()); } - public void addModelRela_1(DomainResp s2Domain, ModelResp genreModelResp, ModelResp artistModelResp) { + public void addModelRela_1( + DomainResp s2Domain, ModelResp genreModelResp, ModelResp artistModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); @@ -248,9 +251,11 @@ public class CspiderDemo extends S2BaseDemo { modelRelaService.save(modelRelaReq, user); } - public void addModelRela_2(DomainResp s2Domain, ModelResp filesModelResp, ModelResp artistModelResp) { + public void addModelRela_2( + DomainResp s2Domain, ModelResp filesModelResp, ModelResp artistModelResp) { List joinConditions = Lists.newArrayList(); - joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); + joinConditions.add( + new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); modelRelaReq.setDomainId(s2Domain.getId()); modelRelaReq.setFromModelId(filesModelResp.getId()); @@ -260,9 +265,11 @@ public class CspiderDemo extends S2BaseDemo { modelRelaService.save(modelRelaReq, user); } - public void addModelRela_3(DomainResp s2Domain, ModelResp songModelResp, ModelResp artistModelResp) { + public void addModelRela_3( + DomainResp s2Domain, ModelResp songModelResp, ModelResp artistModelResp) { List joinConditions = Lists.newArrayList(); - joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); + joinConditions.add( + new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); modelRelaReq.setDomainId(s2Domain.getId()); modelRelaReq.setFromModelId(songModelResp.getId()); @@ -272,7 +279,8 @@ public class CspiderDemo extends S2BaseDemo { modelRelaService.save(modelRelaReq, user); } - public void addModelRela_4(DomainResp s2Domain, ModelResp songModelResp, ModelResp genreModelResp) { + public void addModelRela_4( + DomainResp s2Domain, ModelResp songModelResp, ModelResp genreModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); @@ -284,7 +292,8 @@ public class CspiderDemo extends S2BaseDemo { modelRelaService.save(modelRelaReq, user); } - public void addModelRela_5(DomainResp s2Domain, ModelResp songModelResp, ModelResp filesModelResp) { + public void addModelRela_5( + DomainResp s2Domain, ModelResp songModelResp, ModelResp filesModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("f_id", "f_id", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); @@ -300,5 +309,4 @@ public class CspiderDemo extends S2BaseDemo { List ids = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); metricService.batchPublish(ids, User.getFakeUser()); } - } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/DuSQLDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/DuSQLDemo.java index 677e814fd..e139e3347 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/DuSQLDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/DuSQLDemo.java @@ -10,26 +10,26 @@ import com.tencent.supersonic.chat.server.agent.LLMParserTool; import com.tencent.supersonic.common.pojo.JoinCondition; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; -import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.DataSetDetail; -import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; -import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; +import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -60,7 +60,6 @@ public class DuSQLDemo extends S2BaseDemo { } catch (Exception e) { log.error("Failed to add bench mark demo data", e); } - } @Override @@ -80,7 +79,7 @@ public class DuSQLDemo extends S2BaseDemo { domainService.createDomain(domainReq, user); } - //9 + // 9 public void addModel_1() throws Exception { ModelReq modelReq = new ModelReq(); modelReq.setName("公司"); @@ -95,12 +94,14 @@ public class DuSQLDemo extends S2BaseDemo { ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time.name(), 0); - DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams("false", "none"); + DimensionTimeTypeParams dimensionTimeTypeParams = + new DimensionTimeTypeParams("false", "none"); dimension1.setTypeParams(dimensionTimeTypeParams); dimensions.add(dimension1); dimensions.add(new Dim("公司名称", "company_name", DimensionType.categorical.name(), 1)); dimensions.add(new Dim("总部地点", "headquarter_address", DimensionType.categorical.name(), 1)); - dimensions.add(new Dim("公司成立时间", "company_established_time", DimensionType.categorical.name(), 1)); + dimensions.add( + new Dim("公司成立时间", "company_established_time", DimensionType.categorical.name(), 1)); dimensions.add(new Dim("创始人", "founder", DimensionType.categorical.name(), 1)); dimensions.add(new Dim("首席执行官", "ceo", DimensionType.categorical.name(), 1)); modelDetail.setDimensions(dimensions); @@ -116,8 +117,9 @@ public class DuSQLDemo extends S2BaseDemo { modelDetail.setMeasures(measures); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("SELECT imp_date,company_id,company_name,headquarter_address," - + "company_established_time,founder,ceo,annual_turnover,employee_count FROM company"); + modelDetail.setSqlQuery( + "SELECT imp_date,company_id,company_name,headquarter_address," + + "company_established_time,founder,ceo,annual_turnover,employee_count FROM company"); modelReq.setModelDetail(modelDetail); modelService.createModel(modelReq, user); } @@ -137,12 +139,15 @@ public class DuSQLDemo extends S2BaseDemo { ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time.name(), 0); - DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams("false", "none"); + DimensionTimeTypeParams dimensionTimeTypeParams = + new DimensionTimeTypeParams("false", "none"); dimension1.setTypeParams(dimensionTimeTypeParams); dimensions.add(dimension1); dimensions.add(new Dim("品牌名称", "brand_name", DimensionType.categorical.name(), 1)); - dimensions.add(new Dim("品牌成立时间", "brand_established_time", DimensionType.categorical.name(), 1)); - dimensions.add(new Dim("法定代表人", "legal_representative", DimensionType.categorical.name(), 1)); + dimensions.add( + new Dim("品牌成立时间", "brand_established_time", DimensionType.categorical.name(), 1)); + dimensions.add( + new Dim("法定代表人", "legal_representative", DimensionType.categorical.name(), 1)); modelDetail.setDimensions(dimensions); List identifiers = new ArrayList<>(); @@ -155,8 +160,9 @@ public class DuSQLDemo extends S2BaseDemo { modelDetail.setMeasures(measures); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("SELECT imp_date,brand_id,brand_name,brand_established_time," - + "company_id,legal_representative,registered_capital FROM brand"); + modelDetail.setSqlQuery( + "SELECT imp_date,brand_id,brand_name,brand_established_time," + + "company_id,legal_representative,registered_capital FROM brand"); modelReq.setModelDetail(modelDetail); modelService.createModel(modelReq, user); } @@ -176,7 +182,8 @@ public class DuSQLDemo extends S2BaseDemo { ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time.name(), 0); - DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams("false", "none"); + DimensionTimeTypeParams dimensionTimeTypeParams = + new DimensionTimeTypeParams("false", "none"); dimension1.setTypeParams(dimensionTimeTypeParams); dimensions.add(dimension1); modelDetail.setDimensions(dimensions); @@ -194,8 +201,9 @@ public class DuSQLDemo extends S2BaseDemo { modelDetail.setMeasures(measures); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("SELECT imp_date,company_id,brand_id,revenue_proportion," - + "profit_proportion,expenditure_proportion FROM company_revenue"); + modelDetail.setSqlQuery( + "SELECT imp_date,company_id,brand_id,revenue_proportion," + + "profit_proportion,expenditure_proportion FROM company_revenue"); modelReq.setModelDetail(modelDetail); modelService.createModel(modelReq, user); MetricResp metricResp = metricService.getMetric(13L, user); @@ -221,7 +229,8 @@ public class DuSQLDemo extends S2BaseDemo { ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time.name(), 0); - DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams("false", "none"); + DimensionTimeTypeParams dimensionTimeTypeParams = + new DimensionTimeTypeParams("false", "none"); dimension1.setTypeParams(dimensionTimeTypeParams); dimensions.add(dimension1); dimensions.add(new Dim("年份", "year_time", DimensionType.categorical.name(), 1)); @@ -234,16 +243,19 @@ public class DuSQLDemo extends S2BaseDemo { List measures = new ArrayList<>(); measures.add(new Measure("营收", "revenue", AggOperatorEnum.SUM.name(), 1)); measures.add(new Measure("利润", "profit", AggOperatorEnum.SUM.name(), 1)); - measures.add(new Measure("营收同比增长", "revenue_growth_year_on_year", AggOperatorEnum.SUM.name(), 1)); - measures.add(new Measure("利润同比增长", "profit_growth_year_on_year", AggOperatorEnum.SUM.name(), 1)); + measures.add( + new Measure( + "营收同比增长", "revenue_growth_year_on_year", AggOperatorEnum.SUM.name(), 1)); + measures.add( + new Measure("利润同比增长", "profit_growth_year_on_year", AggOperatorEnum.SUM.name(), 1)); modelDetail.setMeasures(measures); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("SELECT imp_date,year_time,brand_id,revenue,profit," - + "revenue_growth_year_on_year,profit_growth_year_on_year FROM company_brand_revenue"); + modelDetail.setSqlQuery( + "SELECT imp_date,year_time,brand_id,revenue,profit," + + "revenue_growth_year_on_year,profit_growth_year_on_year FROM company_brand_revenue"); modelReq.setModelDetail(modelDetail); modelService.createModel(modelReq, user); - } public void addDataSet_1() { @@ -253,11 +265,20 @@ public class DuSQLDemo extends S2BaseDemo { dataSetReq.setDomainId(4L); dataSetReq.setDescription("DuSQL互联网企业数据源相关的指标和维度等"); dataSetReq.setAdmins(Lists.newArrayList("admin")); - List viewModelConfigs = Lists.newArrayList( - new DataSetModelConfig(9L, Lists.newArrayList(16L, 17L, 18L, 19L, 20L), Lists.newArrayList(10L, 11L)), - new DataSetModelConfig(10L, Lists.newArrayList(21L, 22L, 23L), Lists.newArrayList(12L)), - new DataSetModelConfig(11L, Lists.newArrayList(), Lists.newArrayList(13L, 14L, 15L)), - new DataSetModelConfig(12L, Lists.newArrayList(24L), Lists.newArrayList(16L, 17L, 18L, 19L))); + List viewModelConfigs = + Lists.newArrayList( + new DataSetModelConfig( + 9L, + Lists.newArrayList(16L, 17L, 18L, 19L, 20L), + Lists.newArrayList(10L, 11L)), + new DataSetModelConfig( + 10L, Lists.newArrayList(21L, 22L, 23L), Lists.newArrayList(12L)), + new DataSetModelConfig( + 11L, Lists.newArrayList(), Lists.newArrayList(13L, 14L, 15L)), + new DataSetModelConfig( + 12L, + Lists.newArrayList(24L), + Lists.newArrayList(16L, 17L, 18L, 19L))); DataSetDetail dsDetail = new DataSetDetail(); dsDetail.setDataSetModelConfigs(viewModelConfigs); @@ -276,7 +297,8 @@ public class DuSQLDemo extends S2BaseDemo { public void addModelRela_1() { List joinConditions = Lists.newArrayList(); - joinConditions.add(new JoinCondition("company_id", "company_id", FilterOperatorEnum.EQUALS)); + joinConditions.add( + new JoinCondition("company_id", "company_id", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); modelRelaReq.setDomainId(4L); modelRelaReq.setFromModelId(9L); @@ -288,7 +310,8 @@ public class DuSQLDemo extends S2BaseDemo { public void addModelRela_2() { List joinConditions = Lists.newArrayList(); - joinConditions.add(new JoinCondition("company_id", "company_id", FilterOperatorEnum.EQUALS)); + joinConditions.add( + new JoinCondition("company_id", "company_id", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); modelRelaReq.setDomainId(4L); modelRelaReq.setFromModelId(9L); @@ -341,5 +364,4 @@ public class DuSQLDemo extends S2BaseDemo { log.info("agent:{}", JsonUtil.toString(agent)); agentService.createAgent(agent, User.getFakeUser()); } - } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java index 312336d8e..93011e5a9 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java @@ -97,8 +97,9 @@ public class S2ArtistDemo extends S2BaseDemo { return domainService.createDomain(domainReq, user); } - public ModelResp addModel(DomainResp singerDomain, - DatabaseResp s2Database, TagObjectResp singerTagObject) throws Exception { + public ModelResp addModel( + DomainResp singerDomain, DatabaseResp s2Database, TagObjectResp singerTagObject) + throws Exception { ModelReq modelReq = new ModelReq(); modelReq.setName("歌手库"); modelReq.setBizName("singer"); @@ -118,12 +119,9 @@ public class S2ArtistDemo extends S2BaseDemo { modelDetail.setIdentifiers(identifiers); List dimensions = new ArrayList<>(); - dimensions.add(new Dim("活跃区域", "act_area", - DimensionType.categorical.name(), 1, 1)); - dimensions.add(new Dim("代表作", "song_name", - DimensionType.categorical.name(), 1)); - dimensions.add(new Dim("流派", "genre", - DimensionType.categorical.name(), 1, 1)); + dimensions.add(new Dim("活跃区域", "act_area", DimensionType.categorical.name(), 1, 1)); + dimensions.add(new Dim("代表作", "song_name", DimensionType.categorical.name(), 1)); + dimensions.add(new Dim("流派", "genre", DimensionType.categorical.name(), 1, 1)); modelDetail.setDimensions(dimensions); Measure measure1 = new Measure("播放量", "js_play_cnt", "sum", 1); @@ -131,23 +129,27 @@ public class S2ArtistDemo extends S2BaseDemo { Measure measure3 = new Measure("收藏量", "favor_cnt", "sum", 1); modelDetail.setMeasures(Lists.newArrayList(measure1, measure2, measure3)); modelDetail.setQueryType("sql_query"); - modelDetail.setSqlQuery("select singer_name, act_area, song_name, genre, " - + "js_play_cnt, down_cnt, favor_cnt from singer"); + modelDetail.setSqlQuery( + "select singer_name, act_area, song_name, genre, " + + "js_play_cnt, down_cnt, favor_cnt from singer"); modelReq.setModelDetail(modelDetail); return modelService.createModel(modelReq, user); } private void addTags(ModelResp model) { - addTag(dimensionService.getDimension("act_area", model.getId()).getId(), + addTag( + dimensionService.getDimension("act_area", model.getId()).getId(), TagDefineType.DIMENSION); - addTag(dimensionService.getDimension("song_name", model.getId()).getId(), + addTag( + dimensionService.getDimension("song_name", model.getId()).getId(), TagDefineType.DIMENSION); - addTag(dimensionService.getDimension("genre", model.getId()).getId(), + addTag( + dimensionService.getDimension("genre", model.getId()).getId(), TagDefineType.DIMENSION); - addTag(dimensionService.getDimension("singer_name", model.getId()).getId(), + addTag( + dimensionService.getDimension("singer_name", model.getId()).getId(), TagDefineType.DIMENSION); - addTag(metricService.getMetric(model.getId(), "js_play_cnt").getId(), - TagDefineType.METRIC); + addTag(metricService.getMetric(model.getId(), "js_play_cnt").getId(), TagDefineType.METRIC); } public long addDataSet(DomainResp singerDomain, ModelResp singerModel) { @@ -209,5 +211,4 @@ public class S2ArtistDemo extends S2BaseDemo { agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); agentService.createAgent(agent, User.getFakeUser()); } - } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java index 7ae1f3d6b..e8b833ef9 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.common.service.SystemConfigService; import com.tencent.supersonic.common.util.AESEncryptionUtil; import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.enums.DataType; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; @@ -20,7 +21,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.CanvasService; import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DatabaseService; @@ -49,46 +49,29 @@ public abstract class S2BaseDemo implements CommandLineRunner { protected DatabaseResp demoDatabaseResp; protected User user = User.getFakeUser(); - @Autowired - protected DatabaseService databaseService; - @Autowired - protected DomainService domainService; - @Autowired - protected ModelService modelService; - @Autowired - protected ModelRelaService modelRelaService; - @Autowired - protected DimensionService dimensionService; - @Autowired - protected MetricService metricService; - @Autowired - protected TagMetaService tagMetaService; - @Autowired - protected AuthService authService; - @Autowired - protected DataSetService dataSetService; - @Autowired - protected TermService termService; - @Autowired - protected PluginService pluginService; - @Autowired - protected DataSourceProperties dataSourceProperties; - @Autowired - protected TagObjectService tagObjectService; - @Autowired - protected ChatQueryService chatQueryService; - @Autowired - protected ChatManageService chatManageService; - @Autowired - protected AgentService agentService; - @Autowired - protected SystemConfigService sysParameterService; - @Autowired - protected CanvasService canvasService; - @Autowired - protected DictWordService dictWordService; + @Autowired protected DatabaseService databaseService; + @Autowired protected DomainService domainService; + @Autowired protected ModelService modelService; + @Autowired protected ModelRelaService modelRelaService; + @Autowired protected DimensionService dimensionService; + @Autowired protected MetricService metricService; + @Autowired protected TagMetaService tagMetaService; + @Autowired protected AuthService authService; + @Autowired protected DataSetService dataSetService; + @Autowired protected TermService termService; + @Autowired protected PluginService pluginService; + @Autowired protected DataSourceProperties dataSourceProperties; + @Autowired protected TagObjectService tagObjectService; + @Autowired protected ChatQueryService chatQueryService; + @Autowired protected ChatManageService chatManageService; + @Autowired protected AgentService agentService; + @Autowired protected SystemConfigService sysParameterService; + @Autowired protected CanvasService canvasService; + @Autowired protected DictWordService dictWordService; + @Value("${s2.demo.names:S2VisitsDemo}") protected List demoList; + @Value("${s2.demo.enableLLM:true}") protected boolean demoEnableLlm; @@ -123,7 +106,8 @@ public abstract class S2BaseDemo implements CommandLineRunner { } databaseReq.setUrl(url); databaseReq.setUsername(dataSourceProperties.getUsername()); - databaseReq.setPassword(AESEncryptionUtil.aesEncryptECB(dataSourceProperties.getPassword())); + databaseReq.setPassword( + AESEncryptionUtil.aesEncryptECB(dataSourceProperties.getPassword())); return databaseService.createOrUpdateDatabase(databaseReq, user); } @@ -141,11 +125,15 @@ public abstract class S2BaseDemo implements CommandLineRunner { dataSetModelConfig.setId(modelResp.getId()); MetaFilter metaFilter = new MetaFilter(); metaFilter.setModelIds(Lists.newArrayList(modelResp.getId())); - List metrics = metricService.getMetrics(metaFilter) - .stream().map(MetricResp::getId).collect(Collectors.toList()); + List metrics = + metricService.getMetrics(metaFilter).stream() + .map(MetricResp::getId) + .collect(Collectors.toList()); dataSetModelConfig.setMetrics(metrics); - List dimensions = dimensionService.getDimensions(metaFilter) - .stream().map(DimensionResp::getId).collect(Collectors.toList()); + List dimensions = + dimensionService.getDimensions(metaFilter).stream() + .map(DimensionResp::getId) + .collect(Collectors.toList()); dataSetModelConfig.setMetrics(metrics); dataSetModelConfig.setDimensions(dimensions); dataSetModelConfigs.add(dataSetModelConfig); @@ -175,5 +163,4 @@ public abstract class S2BaseDemo implements CommandLineRunner { protected void updateQueryScore(Integer queryId) { chatManageService.updateFeedback(queryId, 5, ""); } - } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java index 273ee8e49..0f89d6e29 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java @@ -86,7 +86,7 @@ public class S2VisitsDemo extends S2BaseDemo { addModelRela_2(s2Domain, userModel, stayTimeModel); addTags(userModel); - //create metrics and dimensions + // create metrics and dimensions DimensionResp departmentDimension = getDimension("department", userModel); MetricResp metricUv = addMetric_uv(pvUvModel, departmentDimension); MetricResp metricPv = getMetric("pv", pvUvModel); @@ -98,21 +98,21 @@ public class S2VisitsDemo extends S2BaseDemo { updateMetric(stayTimeModel, departmentDimension, userDimension); updateMetric_pv(pvUvModel, departmentDimension, userDimension, metricPv); - //create data set + // create data set DataSetResp s2DataSet = addDataSet(s2Domain); addAuthGroup_1(stayTimeModel); addAuthGroup_2(stayTimeModel); - //create terms and plugin + // create terms and plugin addTerm(s2Domain); addTerm_1(s2Domain); addPlugin(s2DataSet); addPlugin_1(); - //load dict word + // load dict word loadDictWord(); - //create agent + // create agent Integer agentId = addAgent(s2DataSet.getId()); addSampleChats(agentId); updateQueryScore(1); @@ -150,12 +150,13 @@ public class S2VisitsDemo extends S2BaseDemo { agent.setDescription("帮助您用自然语言查询指标，支持时间限定、条件筛选、下钻维度以及聚合统计"); agent.setStatus(1); agent.setEnableSearch(1); - agent.setExamples(Lists.newArrayList( - "超音数访问次数", - "近15天超音数访问次数汇总", - "按部门统计超音数的访问人数", - "对比alice和lucy的停留时长", - "超音数访问次数最高的部门")); + agent.setExamples( + Lists.newArrayList( + "超音数访问次数", + "近15天超音数访问次数汇总", + "按部门统计超音数的访问人数", + "对比alice和lucy的停留时长", + "超音数访问次数最高的部门")); AgentConfig agentConfig = new AgentConfig(); RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); @@ -188,8 +189,9 @@ public class S2VisitsDemo extends S2BaseDemo { return domainService.createDomain(domainReq, user); } - public ModelResp addModel_1(DomainResp s2Domain, DatabaseResp s2Database, - TagObjectResp s2TagObject) throws Exception { + public ModelResp addModel_1( + DomainResp s2Domain, DatabaseResp s2Database, TagObjectResp s2TagObject) + throws Exception { ModelReq modelReq = new ModelReq(); modelReq.setName("用户部门"); modelReq.setBizName("user_department"); @@ -207,8 +209,7 @@ public class S2VisitsDemo extends S2BaseDemo { modelDetail.setIdentifiers(identifiers); List dimensions = new ArrayList<>(); - dimensions.add(new Dim("部门", "department", - DimensionType.categorical.name(), 1)); + dimensions.add(new Dim("部门", "department", DimensionType.categorical.name(), 1)); modelDetail.setDimensions(dimensions); List fields = Lists.newArrayList(); fields.add(Field.builder().fieldName("user_name").dataType("Varchar").build()); @@ -258,8 +259,9 @@ public class S2VisitsDemo extends S2BaseDemo { fields.add(Field.builder().fieldName("pv").dataType("Long").build()); fields.add(Field.builder().fieldName("user_id").dataType("Varchar").build()); modelDetail.setFields(fields); - modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, " - + "user_name as user_id FROM s2_pv_uv_statis"); + modelDetail.setSqlQuery( + "SELECT imp_date, user_name, page, 1 as pv, " + + "user_name as user_id FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelReq.setModelDetail(modelDetail); return modelService.createModel(modelReq, user); @@ -300,13 +302,15 @@ public class S2VisitsDemo extends S2BaseDemo { fields.add(Field.builder().fieldName("page").dataType("Varchar").build()); fields.add(Field.builder().fieldName("stay_hours").dataType("Double").build()); modelDetail.setFields(fields); - modelDetail.setSqlQuery("select imp_date,user_name,stay_hours,page from s2_stay_time_statis"); + modelDetail.setSqlQuery( + "select imp_date,user_name,stay_hours,page from s2_stay_time_statis"); modelDetail.setQueryType("sql_query"); modelReq.setModelDetail(modelDetail); return modelService.createModel(modelReq, user); } - public void addModelRela_1(DomainResp s2Domain, ModelResp userDepartmentModel, ModelResp pvUvModel) { + public void addModelRela_1( + DomainResp s2Domain, ModelResp userDepartmentModel, ModelResp pvUvModel) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("user_name", "user_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); @@ -318,7 +322,8 @@ public class S2VisitsDemo extends S2BaseDemo { modelRelaService.save(modelRelaReq, user); } - public void addModelRela_2(DomainResp s2Domain, ModelResp userDepartmentModel, ModelResp stayTimeModel) { + public void addModelRela_2( + DomainResp s2Domain, ModelResp userDepartmentModel, ModelResp stayTimeModel) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("user_name", "user_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); @@ -331,11 +336,13 @@ public class S2VisitsDemo extends S2BaseDemo { } private void addTags(ModelResp model) { - addTag(dimensionService.getDimension("department", model.getId()).getId(), + addTag( + dimensionService.getDimension("department", model.getId()).getId(), TagDefineType.DIMENSION); } - public void updateDimension(ModelResp stayTimeModel, DimensionResp pageDimension) throws Exception { + public void updateDimension(ModelResp stayTimeModel, DimensionResp pageDimension) + throws Exception { DimensionReq dimensionReq = new DimensionReq(); dimensionReq.setType(DimensionType.categorical.name()); dimensionReq.setId(pageDimension.getId()); @@ -351,10 +358,10 @@ public class S2VisitsDemo extends S2BaseDemo { dimensionService.updateDimension(dimensionReq, user); } - public void updateMetric(ModelResp stayTimeModel, DimensionResp departmentDimension, - DimensionResp userDimension) throws Exception { - MetricResp stayHoursMetric = - metricService.getMetric(stayTimeModel.getId(), "stay_hours"); + public void updateMetric( + ModelResp stayTimeModel, DimensionResp departmentDimension, DimensionResp userDimension) + throws Exception { + MetricResp stayHoursMetric = metricService.getMetric(stayTimeModel.getId(), "stay_hours"); MetricReq metricReq = new MetricReq(); metricReq.setModelId(stayTimeModel.getId()); metricReq.setId(stayHoursMetric.getId()); @@ -366,19 +373,25 @@ public class S2VisitsDemo extends S2BaseDemo { MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams(); metricTypeParams.setExpr("s2_stay_time_statis_stay_hours"); List measures = new ArrayList<>(); - MeasureParam measure = new MeasureParam("s2_stay_time_statis_stay_hours", - "", AggOperatorEnum.SUM.getOperator()); + MeasureParam measure = + new MeasureParam( + "s2_stay_time_statis_stay_hours", "", AggOperatorEnum.SUM.getOperator()); measures.add(measure); metricTypeParams.setMeasures(measures); metricReq.setMetricDefineByMeasureParams(metricTypeParams); metricReq.setMetricDefineType(MetricDefineType.MEASURE); - metricReq.setRelateDimension(getRelateDimension( - Lists.newArrayList(departmentDimension.getId(), userDimension.getId()))); + metricReq.setRelateDimension( + getRelateDimension( + Lists.newArrayList(departmentDimension.getId(), userDimension.getId()))); metricService.updateMetric(metricReq, user); } - public void updateMetric_pv(ModelResp pvUvModel, DimensionResp departmentDimension, - DimensionResp userDimension, MetricResp metricPv) throws Exception { + public void updateMetric_pv( + ModelResp pvUvModel, + DimensionResp departmentDimension, + DimensionResp userDimension, + MetricResp metricPv) + throws Exception { MetricReq metricReq = new MetricReq(); metricReq.setModelId(pvUvModel.getId()); metricReq.setId(metricPv.getId()); @@ -388,18 +401,20 @@ public class S2VisitsDemo extends S2BaseDemo { MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams(); metricTypeParams.setExpr("s2_pv_uv_statis_pv"); List measures = new ArrayList<>(); - MeasureParam measure = new MeasureParam("s2_pv_uv_statis_pv", - "", AggOperatorEnum.SUM.getOperator()); + MeasureParam measure = + new MeasureParam("s2_pv_uv_statis_pv", "", AggOperatorEnum.SUM.getOperator()); measures.add(measure); metricTypeParams.setMeasures(measures); metricReq.setMetricDefineByMeasureParams(metricTypeParams); metricReq.setMetricDefineType(MetricDefineType.MEASURE); - metricReq.setRelateDimension(getRelateDimension( - Lists.newArrayList(departmentDimension.getId(), userDimension.getId()))); + metricReq.setRelateDimension( + getRelateDimension( + Lists.newArrayList(departmentDimension.getId(), userDimension.getId()))); metricService.updateMetric(metricReq, user); } - public MetricResp addMetric_uv(ModelResp uvModel, DimensionResp departmentDimension) throws Exception { + public MetricResp addMetric_uv(ModelResp uvModel, DimensionResp departmentDimension) + throws Exception { MetricReq metricReq = new MetricReq(); metricReq.setModelId(uvModel.getId()); metricReq.setName("访问用户数"); @@ -414,13 +429,17 @@ public class S2VisitsDemo extends S2BaseDemo { metricTypeParams.setFields(fieldParams); metricReq.setMetricDefineByFieldParams(metricTypeParams); metricReq.setMetricDefineType(MetricDefineType.FIELD); - metricReq.setRelateDimension(getRelateDimension( - Lists.newArrayList(departmentDimension.getId()))); + metricReq.setRelateDimension( + getRelateDimension(Lists.newArrayList(departmentDimension.getId()))); return metricService.createMetric(metricReq, user); } - public MetricResp addMetric_pv_avg(MetricResp metricPv, MetricResp metricUv, - DimensionResp departmentDimension, ModelResp pvModel) throws Exception { + public MetricResp addMetric_pv_avg( + MetricResp metricPv, + MetricResp metricUv, + DimensionResp departmentDimension, + ModelResp pvModel) + throws Exception { MetricReq metricReq = new MetricReq(); metricReq.setModelId(pvModel.getId()); metricReq.setName("人均访问次数"); @@ -439,7 +458,8 @@ public class S2VisitsDemo extends S2BaseDemo { metricTypeParams.setMetrics(metrics); metricReq.setMetricDefineByMetricParams(metricTypeParams); metricReq.setMetricDefineType(MetricDefineType.METRIC); - metricReq.setRelateDimension(getRelateDimension(Lists.newArrayList(departmentDimension.getId()))); + metricReq.setRelateDimension( + getRelateDimension(Lists.newArrayList(departmentDimension.getId()))); return metricService.createMetric(metricReq, user); } @@ -555,5 +575,4 @@ public class S2VisitsDemo extends S2BaseDemo { private void loadDictWord() { dictWordService.loadDictWord(); } - } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/SmallTalkDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/SmallTalkDemo.java index 023a6f75a..206c6b605 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/SmallTalkDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/SmallTalkDemo.java @@ -5,7 +5,6 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.agent.AgentConfig; - import com.tencent.supersonic.chat.server.agent.MultiTurnConfig; import lombok.extern.slf4j.Slf4j; import org.springframework.core.annotation.Order; @@ -27,8 +26,7 @@ public class SmallTalkDemo extends S2BaseDemo { agent.setEnableSearch(0); AgentConfig agentConfig = new AgentConfig(); agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); - agent.setExamples(Lists.newArrayList("如何才能变帅", - "如何才能赚更多钱", "如何才能世界和平")); + agent.setExamples(Lists.newArrayList("如何才能变帅", "如何才能赚更多钱", "如何才能世界和平")); MultiTurnConfig multiTurnConfig = new MultiTurnConfig(); multiTurnConfig.setEnableMultiTurn(true); agent.setMultiTurnConfig(multiTurnConfig); @@ -38,9 +36,8 @@ public class SmallTalkDemo extends S2BaseDemo { @Override boolean checkNeedToRun() { - List agentNames = agentService.getAgents() - .stream().map(Agent::getName).collect(Collectors.toList()); + List agentNames = + agentService.getAgents().stream().map(Agent::getName).collect(Collectors.toList()); return !agentNames.contains("来闲聊"); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/BaseApplication.java b/launchers/standalone/src/test/java/com/tencent/supersonic/BaseApplication.java index 1b854a67b..3f4436d2d 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/BaseApplication.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/BaseApplication.java @@ -3,6 +3,4 @@ package com.tencent.supersonic; import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest(classes = {StandaloneLauncher.class}) -public class BaseApplication { - -} +public class BaseApplication {} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java index 31991b864..631702de5 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java @@ -3,12 +3,12 @@ package com.tencent.supersonic.chat; import com.tencent.supersonic.BaseApplication; import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatQueryService; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.util.DataUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -26,23 +26,23 @@ public class BaseTest extends BaseApplication { protected final String endDay = LocalDate.now().plusDays(-1).toString(); protected final String period = "DAY"; - @Autowired - protected ChatQueryService chatQueryService; - @Autowired - protected AgentService agentService; + @Autowired protected ChatQueryService chatQueryService; + @Autowired protected AgentService agentService; - protected QueryResult submitMultiTurnChat(String queryText, Integer agentId, Integer chatId) throws Exception { + protected QueryResult submitMultiTurnChat(String queryText, Integer agentId, Integer chatId) + throws Exception { ParseResp parseResp = submitParse(queryText, agentId, chatId); SemanticParseInfo semanticParseInfo = parseResp.getSelectedParses().get(0); - ChatExecuteReq request = ChatExecuteReq.builder() - .queryText(parseResp.getQueryText()) - .user(DataUtils.getUser()) - .parseId(semanticParseInfo.getId()) - .queryId(parseResp.getQueryId()) - .chatId(chatId) - .saveAnswer(true) - .build(); + ChatExecuteReq request = + ChatExecuteReq.builder() + .queryText(parseResp.getQueryText()) + .user(DataUtils.getUser()) + .parseId(semanticParseInfo.getId()) + .queryId(parseResp.getQueryId()) + .chatId(chatId) + .saveAnswer(true) + .build(); QueryResult queryResult = chatQueryService.performExecution(request); queryResult.setChatContext(semanticParseInfo); return queryResult; @@ -53,15 +53,16 @@ public class BaseTest extends BaseApplication { ParseResp parseResp = submitParse(queryText, agentId, chatId); SemanticParseInfo parseInfo = parseResp.getSelectedParses().get(0); - ChatExecuteReq request = ChatExecuteReq.builder() - .queryText(parseResp.getQueryText()) - .user(DataUtils.getUser()) - .parseId(parseInfo.getId()) - .agentId(agentId) - .chatId(chatId) - .queryId(parseResp.getQueryId()) - .saveAnswer(false) - .build(); + ChatExecuteReq request = + ChatExecuteReq.builder() + .queryText(parseResp.getQueryText()) + .user(DataUtils.getUser()) + .parseId(parseInfo.getId()) + .agentId(agentId) + .chatId(chatId) + .queryId(parseResp.getQueryId()) + .saveAnswer(false) + .build(); QueryResult result = chatQueryService.performExecution(request); result.setChatContext(parseInfo); @@ -75,10 +76,16 @@ public class BaseTest extends BaseApplication { } protected void assertSchemaElements(Set expected, Set actual) { - Set expectedNames = expected.stream().map(s -> s.getName()) - .filter(s -> s != null).collect(Collectors.toSet()); - Set actualNames = actual.stream().map(s -> s.getName()) - .filter(s -> s != null).collect(Collectors.toSet()); + Set expectedNames = + expected.stream() + .map(s -> s.getName()) + .filter(s -> s != null) + .collect(Collectors.toSet()); + Set actualNames = + actual.stream() + .map(s -> s.getName()) + .filter(s -> s != null) + .collect(Collectors.toSet()); assertEquals(expectedNames, actualNames); } @@ -94,10 +101,10 @@ public class BaseTest extends BaseApplication { assertSchemaElements(expectedParseInfo.getMetrics(), actualParseInfo.getMetrics()); assertSchemaElements(expectedParseInfo.getDimensions(), actualParseInfo.getDimensions()); - assertEquals(expectedParseInfo.getDimensionFilters(), actualParseInfo.getDimensionFilters()); + assertEquals( + expectedParseInfo.getDimensionFilters(), actualParseInfo.getDimensionFilters()); assertEquals(expectedParseInfo.getMetricFilters(), actualParseInfo.getMetricFilters()); assertEquals(expectedParseInfo.getDateInfo(), actualParseInfo.getDateInfo()); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java index 77aa6b707..4146e6cc4 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java @@ -32,13 +32,16 @@ public class DetailTest extends BaseTest { expectedParseInfo.setQueryType(QueryType.DETAIL); expectedParseInfo.setAggType(AggregateTypeEnum.NONE); - QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, - "周杰伦", "歌手名", 8L); + QueryFilter dimensionFilter = + DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 8L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); - expectedParseInfo.getDimensions().addAll(Lists.newArrayList( - SchemaElement.builder().name("流派").build(), - SchemaElement.builder().name("代表作").build())); + expectedParseInfo + .getDimensions() + .addAll( + Lists.newArrayList( + SchemaElement.builder().name("流派").build(), + SchemaElement.builder().name("代表作").build())); assertQueryResult(expectedResult, actualResult); } @@ -55,17 +58,19 @@ public class DetailTest extends BaseTest { expectedParseInfo.setQueryType(QueryType.DETAIL); expectedParseInfo.setAggType(AggregateTypeEnum.NONE); - QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, - "周杰伦", "歌手名", 8L); + QueryFilter dimensionFilter = + DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 8L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); expectedParseInfo.getMetrics().add(SchemaElement.builder().name("播放量").build()); - expectedParseInfo.getDimensions().addAll(Lists.newArrayList( - SchemaElement.builder().name("歌手名").build(), - SchemaElement.builder().name("活跃区域").build(), - SchemaElement.builder().name("流派").build(), - SchemaElement.builder().name("代表作").build() - )); + expectedParseInfo + .getDimensions() + .addAll( + Lists.newArrayList( + SchemaElement.builder().name("歌手名").build(), + SchemaElement.builder().name("活跃区域").build(), + SchemaElement.builder().name("流派").build(), + SchemaElement.builder().name("代表作").build())); assertQueryResult(expectedResult, actualResult); } @@ -82,19 +87,20 @@ public class DetailTest extends BaseTest { expectedParseInfo.setQueryType(QueryType.DETAIL); expectedParseInfo.setAggType(AggregateTypeEnum.NONE); - QueryFilter dimensionFilter = DataUtils.getFilter("genre", FilterOperatorEnum.EQUALS, - "国风", "流派", 7L); + QueryFilter dimensionFilter = + DataUtils.getFilter("genre", FilterOperatorEnum.EQUALS, "国风", "流派", 7L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); expectedParseInfo.getMetrics().add(SchemaElement.builder().name("播放量").build()); - expectedParseInfo.getDimensions().addAll(Lists.newArrayList( - SchemaElement.builder().name("歌手名").build(), - SchemaElement.builder().name("活跃区域").build(), - SchemaElement.builder().name("流派").build(), - SchemaElement.builder().name("代表作").build() - )); + expectedParseInfo + .getDimensions() + .addAll( + Lists.newArrayList( + SchemaElement.builder().name("歌手名").build(), + SchemaElement.builder().name("活跃区域").build(), + SchemaElement.builder().name("流派").build(), + SchemaElement.builder().name("代表作").build())); assertQueryResult(expectedResult, actualResult); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java index 83d7d9c12..0689b6726 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.chat; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricFilterQuery; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricGroupByQuery; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricModelQuery; @@ -38,16 +38,20 @@ public class MetricTest extends BaseTest { expectedParseInfo.setAggType(NONE); expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); + expectedParseInfo + .getDimensionFilters() + .add( + DataUtils.getFilter( + "user_name", FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } - //@Test + // @Test public void testMetricDomain() throws Exception { QueryResult actualResult = submitNewChat("超音数总访问次数", DataUtils.metricAgentId); @@ -58,7 +62,8 @@ public class MetricTest extends BaseTest { expectedResult.setQueryMode(MetricModelQuery.QUERY_MODE); expectedParseInfo.setAggType(NONE); expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); @@ -78,7 +83,8 @@ public class MetricTest extends BaseTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); @@ -99,10 +105,12 @@ public class MetricTest extends BaseTest { List list = new ArrayList<>(); list.add("alice"); list.add("lucy"); - QueryFilter dimensionFilter = DataUtils.getFilter("user_name", FilterOperatorEnum.IN, list, "用户", 2L); + QueryFilter dimensionFilter = + DataUtils.getFilter("user_name", FilterOperatorEnum.IN, list, "用户", 2L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); @@ -142,7 +150,8 @@ public class MetricTest extends BaseTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); @@ -154,7 +163,8 @@ public class MetricTest extends BaseTest { DateFormat textFormat = new SimpleDateFormat("yyyy年mm月dd日"); String dateStr = textFormat.format(format.parse(startDay)); - QueryResult actualResult = submitNewChat(String.format("想知道%salice的访问次数", dateStr), DataUtils.metricAgentId); + QueryResult actualResult = + submitNewChat(String.format("想知道%salice的访问次数", dateStr), DataUtils.metricAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -164,13 +174,16 @@ public class MetricTest extends BaseTest { expectedParseInfo.setAggType(NONE); expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); + expectedParseInfo + .getDimensionFilters() + .add( + DataUtils.getFilter( + "user_name", FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java index f493d9934..0b96a6e8d 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricFilterQuery; import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Order; @@ -17,8 +17,9 @@ public class MultiTurnsTest extends BaseTest { @Test @Order(1) public void queryTest_01() throws Exception { - QueryResult actualResult = submitMultiTurnChat("alice的访问次数", - DataUtils.metricAgentId, DataUtils.MULTI_TURNS_CHAT_ID); + QueryResult actualResult = + submitMultiTurnChat( + "alice的访问次数", DataUtils.metricAgentId, DataUtils.MULTI_TURNS_CHAT_ID); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -29,10 +30,14 @@ public class MultiTurnsTest extends BaseTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); + expectedParseInfo + .getDimensionFilters() + .add( + DataUtils.getFilter( + "user_name", FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); @@ -41,8 +46,9 @@ public class MultiTurnsTest extends BaseTest { @Test @Order(2) public void queryTest_02() throws Exception { - QueryResult actualResult = submitMultiTurnChat("停留时长呢", DataUtils.metricAgentId, - DataUtils.MULTI_TURNS_CHAT_ID); + QueryResult actualResult = + submitMultiTurnChat( + "停留时长呢", DataUtils.metricAgentId, DataUtils.MULTI_TURNS_CHAT_ID); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -53,10 +59,14 @@ public class MultiTurnsTest extends BaseTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("停留时长")); - expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); + expectedParseInfo + .getDimensionFilters() + .add( + DataUtils.getFilter( + "user_name", FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); @@ -65,8 +75,9 @@ public class MultiTurnsTest extends BaseTest { @Test @Order(3) public void queryTest_03() throws Exception { - QueryResult actualResult = submitMultiTurnChat("lucy的如何", DataUtils.metricAgentId, - DataUtils.MULTI_TURNS_CHAT_ID); + QueryResult actualResult = + submitMultiTurnChat( + "lucy的如何", DataUtils.metricAgentId, DataUtils.MULTI_TURNS_CHAT_ID); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -77,13 +88,14 @@ public class MultiTurnsTest extends BaseTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("停留时长")); - expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "lucy", "用户", 2L)); + expectedParseInfo + .getDimensionFilters() + .add(DataUtils.getFilter("user_name", FilterOperatorEnum.EQUALS, "lucy", "用户", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java b/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java index dbfab71c0..5faa3d6b5 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java @@ -95,7 +95,10 @@ public class Text2SQLEval extends BaseTest { QueryResult result = submitNewChat("过去半个月核心用户的总停留时长", agentId); assert result.getQueryColumns().size() >= 1; assert result.getQueryColumns().stream() - .filter(c -> c.getName().contains("停留时长")).collect(Collectors.toList()).size() == 1; + .filter(c -> c.getName().contains("停留时长")) + .collect(Collectors.toList()) + .size() + == 1; assert result.getQueryResults().size() >= 1; } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java index 4037465ec..5732f6d47 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java @@ -12,9 +12,9 @@ import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO; import com.tencent.supersonic.headless.server.persistence.repository.DomainRepository; -import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.util.DataUtils; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -27,11 +27,9 @@ import static java.time.LocalDate.now; public class BaseTest extends BaseApplication { - @Autowired - protected SemanticLayerService semanticLayerService; + @Autowired protected SemanticLayerService semanticLayerService; - @Autowired - private DomainRepository domainRepository; + @Autowired private DomainRepository domainRepository; protected SemanticQueryResp queryBySql(String sql) throws Exception { return queryBySql(sql, User.getFakeUser()); @@ -81,8 +79,7 @@ public class BaseTest extends BaseApplication { return queryStructReq; } - protected QueryStructReq buildQueryStructReq(List groups, - Aggregator aggregator) { + protected QueryStructReq buildQueryStructReq(List groups, Aggregator aggregator) { QueryStructReq queryStructReq = new QueryStructReq(); for (Long modelId : DataUtils.getMetricAgentIModelIds()) { queryStructReq.addModelId(modelId); @@ -108,5 +105,4 @@ public class BaseTest extends BaseApplication { domainDO.setIsOpen(0); domainRepository.updateDomain(domainDO); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java index 9eeaf0ba8..a40329100 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java @@ -18,13 +18,10 @@ import java.util.Arrays; import java.util.Date; import java.util.List; - public class DictTest extends BaseTest { - @Autowired - private DictConfMapper confMapper; + @Autowired private DictConfMapper confMapper; - @Autowired - private DictTaskService taskService; + @Autowired private DictTaskService taskService; @Test public void insertConf() { @@ -83,12 +80,15 @@ public class DictTest extends BaseTest { void testAddTask() { editConf(); DictConfDO confDODb = confMapper.selectById(1L); - DictSingleTaskReq dictTask = DictSingleTaskReq.builder().itemId(confDODb.getItemId()) - .type(TypeEnums.DIMENSION).build(); + DictSingleTaskReq dictTask = + DictSingleTaskReq.builder() + .itemId(confDODb.getItemId()) + .type(TypeEnums.DIMENSION) + .build(); taskService.addDictTask(dictTask, null); - DictSingleTaskReq taskReq = DictSingleTaskReq.builder().itemId(3L).type(TypeEnums.DIMENSION).build(); + DictSingleTaskReq taskReq = + DictSingleTaskReq.builder().itemId(3L).type(TypeEnums.DIMENSION).build(); taskService.deleteDictTask(taskReq, null); System.out.println(); } - -} \ No newline at end of file +} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java index 345de8549..0910e2fc0 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java @@ -1,8 +1,5 @@ package com.tencent.supersonic.headless; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.auth.authentication.strategy.FakeUserStrategy; import com.tencent.supersonic.headless.server.task.FlightServerInitTask; @@ -19,36 +16,37 @@ import org.apache.arrow.memory.RootAllocator; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + @Slf4j public class FlightSqlTest extends BaseTest { - - @Autowired - private FlightServerInitTask flightSqlListener; - @Autowired - private FakeUserStrategy fakeUserStrategy; + @Autowired private FlightServerInitTask flightSqlListener; + @Autowired private FakeUserStrategy fakeUserStrategy; @Test void test01() throws Exception { startServer(); String host = flightSqlListener.getHost(); Integer port = flightSqlListener.getPort(); - FlightSqlClient sqlClient = new FlightSqlClient( - FlightClient.builder(new RootAllocator(Integer.MAX_VALUE), Location.forGrpcInsecure(host, port)) - .build()); + FlightSqlClient sqlClient = + new FlightSqlClient( + FlightClient.builder( + new RootAllocator(Integer.MAX_VALUE), + Location.forGrpcInsecure(host, port)) + .build()); CallHeaders headers = new FlightCallHeaders(); headers.insert("dataSetId", "1"); headers.insert("name", "admin"); headers.insert("password", "admin"); HeaderCallOption headerOption = new HeaderCallOption(headers); - try (final FlightSqlClient.PreparedStatement preparedStatement = sqlClient.prepare( - "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门", - headerOption)) { + try (final FlightSqlClient.PreparedStatement preparedStatement = + sqlClient.prepare( + "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门", headerOption)) { final FlightInfo info = preparedStatement.execute(); - FlightStream stream = sqlClient.getStream(info - .getEndpoints() - .get(0).getTicket()); + FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket()); int rowCnt = 0; int colCnt = 0; while (stream.next()) { @@ -69,9 +67,12 @@ public class FlightSqlTest extends BaseTest { startServer(); String host = flightSqlListener.getHost(); Integer port = flightSqlListener.getPort(); - FlightSqlClient sqlClient = new FlightSqlClient( - FlightClient.builder(new RootAllocator(Integer.MAX_VALUE), Location.forGrpcInsecure(host, port)) - .build()); + FlightSqlClient sqlClient = + new FlightSqlClient( + FlightClient.builder( + new RootAllocator(Integer.MAX_VALUE), + Location.forGrpcInsecure(host, port)) + .build()); CallHeaders headers = new FlightCallHeaders(); headers.insert("dataSetId", "1"); @@ -79,12 +80,11 @@ public class FlightSqlTest extends BaseTest { headers.insert("password", "admin"); HeaderCallOption headerOption = new HeaderCallOption(headers); try { - FlightInfo flightInfo = sqlClient.execute( - "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门", - headerOption); - FlightStream stream = sqlClient.getStream(flightInfo - .getEndpoints() - .get(0).getTicket()); + FlightInfo flightInfo = + sqlClient.execute( + "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门", + headerOption); + FlightStream stream = sqlClient.getStream(flightInfo.getEndpoints().get(0).getTicket()); int rowCnt = 0; int colCnt = 0; while (stream.next()) { diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java index 9e0f4755f..b5a6814ab 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java @@ -14,8 +14,7 @@ import java.util.Collections; public class MetaDiscoveryTest extends BaseTest { - @Autowired - protected ChatLayerService chatLayerService; + @Autowired protected ChatLayerService chatLayerService; @Test public void testGetMapMeta() throws Exception { diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java index f688c7909..9a4e6c989 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java @@ -15,8 +15,7 @@ import java.util.stream.Collectors; public class ModelSchemaTest extends BaseTest { - @Autowired - private ModelService modelService; + @Autowired private ModelService modelService; @Test void testGetUnAvailableItem() { @@ -25,9 +24,11 @@ public class ModelSchemaTest extends BaseTest { fieldRemovedReq.setFields(Lists.newArrayList("pv")); UnAvailableItemResp unAvailableItemResp = modelService.getUnAvailableItem(fieldRemovedReq); List expectedUnAvailableMetricId = Lists.newArrayList(1L, 4L); - List actualUnAvailableMetricId = unAvailableItemResp.getMetricResps() - .stream().map(MetricResp::getId).sorted(Comparator.naturalOrder()).collect(Collectors.toList()); + List actualUnAvailableMetricId = + unAvailableItemResp.getMetricResps().stream() + .map(MetricResp::getId) + .sorted(Comparator.naturalOrder()) + .collect(Collectors.toList()); Assertions.assertEquals(expectedUnAvailableMetricId, actualUnAvailableMetricId); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java index d44d271cd..a5ef3e3a0 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java @@ -15,8 +15,7 @@ import static org.junit.Assert.assertThrows; public class QueryByMetricTest extends BaseTest { - @Autowired - protected MetricService metricService; + @Autowired protected MetricService metricService; @Test public void testWithMetricAndDimensionBizNames() throws Exception { @@ -51,7 +50,8 @@ public class QueryByMetricTest extends BaseTest { queryMetricReq.setDomainId(2L); queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); - assertThrows(IllegalArgumentException.class, + assertThrows( + IllegalArgumentException.class, () -> queryByMetric(queryMetricReq, User.getFakeUser())); } @@ -66,7 +66,8 @@ public class QueryByMetricTest extends BaseTest { Assert.assertEquals(6, queryResp.getResultList().size()); } - private SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception { + private SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) + throws Exception { QueryStructReq convert = metricService.convert(queryMetricReq); return semanticLayerService.queryByReq(convert.convert(), user); } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java index 4723537b6..43bbe8f85 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java @@ -17,7 +17,8 @@ public class QueryBySqlTest extends BaseTest { @Test public void testDetailQuery() throws Exception { - SemanticQueryResp semanticQueryResp = queryBySql("SELECT 用户,访问次数 FROM 超音数PVUV统计 WHERE 用户='alice' "); + SemanticQueryResp semanticQueryResp = + queryBySql("SELECT 用户,访问次数 FROM 超音数PVUV统计 WHERE 用户='alice' "); assertEquals(2, semanticQueryResp.getColumns().size()); QueryColumn firstColumn = semanticQueryResp.getColumns().get(0); @@ -29,7 +30,8 @@ public class QueryBySqlTest extends BaseTest { @Test public void testSumQuery() throws Exception { - SemanticQueryResp semanticQueryResp = queryBySql("SELECT SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 "); + SemanticQueryResp semanticQueryResp = + queryBySql("SELECT SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 "); assertEquals(1, semanticQueryResp.getColumns().size()); QueryColumn queryColumn = semanticQueryResp.getColumns().get(0); @@ -39,7 +41,8 @@ public class QueryBySqlTest extends BaseTest { @Test public void testGroupByQuery() throws Exception { - SemanticQueryResp result = queryBySql("SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "); + SemanticQueryResp result = + queryBySql("SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "); assertEquals(2, result.getColumns().size()); QueryColumn firstColumn = result.getColumns().get(0); QueryColumn secondColumn = result.getColumns().get(1); @@ -50,8 +53,9 @@ public class QueryBySqlTest extends BaseTest { @Test public void testFilterQuery() throws Exception { - SemanticQueryResp result = queryBySql( - "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 WHERE 部门 ='HR' GROUP BY 部门 "); + SemanticQueryResp result = + queryBySql( + "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 WHERE 部门 ='HR' GROUP BY 部门 "); assertEquals(2, result.getColumns().size()); QueryColumn firstColumn = result.getColumns().get(0); QueryColumn secondColumn = result.getColumns().get(1); @@ -76,13 +80,15 @@ public class QueryBySqlTest extends BaseTest { @Test public void testCacheQuery() throws Exception { queryBySql("SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "); - SemanticQueryResp result2 = queryBySql("SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "); + SemanticQueryResp result2 = + queryBySql("SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "); assertTrue(result2.isUseCache()); } @Test public void testBizNameQuery() throws Exception { - SemanticQueryResp result1 = queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'"); + SemanticQueryResp result1 = + queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'"); SemanticQueryResp result2 = queryBySql("SELECT SUM(访问次数) FROM 超音数PVUV统计 WHERE 部门 ='HR'"); assertEquals(1, result1.getColumns().size()); assertEquals(1, result2.getColumns().size()); @@ -94,15 +100,19 @@ public class QueryBySqlTest extends BaseTest { public void testAuthorization_model() { User alice = DataUtils.getUserAlice(); setDomainNotOpenToAll(); - assertThrows(InvalidPermissionException.class, + assertThrows( + InvalidPermissionException.class, () -> queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'", alice)); } @Test public void testAuthorization_sensitive_metric() throws Exception { User tom = DataUtils.getUserTom(); - assertThrows(InvalidPermissionException.class, - () -> queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom)); + assertThrows( + InvalidPermissionException.class, + () -> + queryBySql( + "SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom)); } @Test @@ -121,5 +131,4 @@ public class QueryBySqlTest extends BaseTest { Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage()); Assertions.assertTrue(semanticQueryResp.getSql().contains("user_name = 'tom'")); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java index f34c7a6b7..4e6d506cc 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java @@ -19,9 +19,11 @@ import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; @@ -44,9 +46,10 @@ public class QueryByStructTest extends BaseTest { @Test public void testDetailQuery() throws Exception { - QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("user_name", "department"), - QueryType.DETAIL); - SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); + QueryStructReq queryStructReq = + buildQueryStructReq(Arrays.asList("user_name", "department"), QueryType.DETAIL); + SemanticQueryResp semanticQueryResp = + semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); assertEquals(3, semanticQueryResp.getColumns().size()); QueryColumn firstColumn = semanticQueryResp.getColumns().get(0); assertEquals("用户", firstColumn.getName()); @@ -60,7 +63,8 @@ public class QueryByStructTest extends BaseTest { @Test public void testSumQuery() throws Exception { QueryStructReq queryStructReq = buildQueryStructReq(null); - SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); + SemanticQueryResp semanticQueryResp = + semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); assertEquals(1, semanticQueryResp.getColumns().size()); QueryColumn queryColumn = semanticQueryResp.getColumns().get(0); assertEquals("访问次数", queryColumn.getName()); @@ -70,7 +74,8 @@ public class QueryByStructTest extends BaseTest { @Test public void testGroupByQuery() throws Exception { QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department")); - SemanticQueryResp result = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); + SemanticQueryResp result = + semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); assertEquals(2, result.getColumns().size()); QueryColumn firstColumn = result.getColumns().get(0); QueryColumn secondColumn = result.getColumns().get(1); @@ -91,7 +96,8 @@ public class QueryByStructTest extends BaseTest { dimensionFilters.add(filter); queryStructReq.setDimensionFilters(dimensionFilters); - SemanticQueryResp result = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); + SemanticQueryResp result = + semanticLayerService.queryByReq(queryStructReq, User.getFakeUser()); assertEquals(2, result.getColumns().size()); QueryColumn firstColumn = result.getColumns().get(0); QueryColumn secondColumn = result.getColumns().get(1); @@ -106,7 +112,8 @@ public class QueryByStructTest extends BaseTest { User alice = DataUtils.getUserAlice(); setDomainNotOpenToAll(); QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department")); - assertThrows(InvalidPermissionException.class, + assertThrows( + InvalidPermissionException.class, () -> semanticLayerService.queryByReq(queryStructReq1, alice)); } @@ -116,8 +123,10 @@ public class QueryByStructTest extends BaseTest { Aggregator aggregator = new Aggregator(); aggregator.setFunc(AggOperatorEnum.SUM); aggregator.setColumn("stay_hours"); - QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"), aggregator); - assertThrows(InvalidPermissionException.class, + QueryStructReq queryStructReq = + buildQueryStructReq(Arrays.asList("department"), aggregator); + assertThrows( + InvalidPermissionException.class, () -> semanticLayerService.queryByReq(queryStructReq, tom)); } @@ -127,10 +136,10 @@ public class QueryByStructTest extends BaseTest { Aggregator aggregator = new Aggregator(); aggregator.setFunc(AggOperatorEnum.SUM); aggregator.setColumn("pv"); - QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator); + QueryStructReq queryStructReq1 = + buildQueryStructReq(Arrays.asList("department"), aggregator); SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom); Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage()); Assertions.assertTrue(semanticQueryResp.getSql().contains("`user_name` = 'tom'")); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryDimensionTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryDimensionTest.java index 117f221a6..829f9c3bf 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryDimensionTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryDimensionTest.java @@ -14,9 +14,9 @@ public class QueryDimensionTest extends BaseTest { queryDimValueReq.setModelId(1L); queryDimValueReq.setBizName("department"); - SemanticQueryResp queryResp = semanticLayerService.queryDimensionValue(queryDimValueReq, User.getFakeUser()); + SemanticQueryResp queryResp = + semanticLayerService.queryDimensionValue(queryDimValueReq, User.getFakeUser()); Assert.assertNotNull(queryResp.getResultList()); Assert.assertEquals(4, queryResp.getResultList().size()); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryRuleTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryRuleTest.java index 4e34f873d..226fd6b29 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryRuleTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryRuleTest.java @@ -19,8 +19,7 @@ import java.util.List; public class QueryRuleTest extends BaseTest { - @Autowired - private QueryRuleService queryRuleService; + @Autowired private QueryRuleService queryRuleService; private User user = User.getFakeUser(); @@ -93,7 +92,8 @@ public class QueryRuleTest extends BaseTest { queryRuleService.addQueryRule(queryRuleReq2, user); QueryRuleFilter queryRuleFilter = new QueryRuleFilter(); - List queryRuleList = queryRuleService.getQueryRuleList(queryRuleFilter, user); + List queryRuleList = + queryRuleService.getQueryRuleList(queryRuleFilter, user); queryRuleList.size(); } -} \ No newline at end of file +} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java index eadb4731f..6e40fff2d 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java @@ -3,29 +3,27 @@ package com.tencent.supersonic.headless; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; + import java.util.List; import java.util.stream.Collectors; public class SchemaAuthTest extends BaseTest { - @Autowired - private DomainService domainService; + @Autowired private DomainService domainService; - @Autowired - private DataSetService dataSetService; + @Autowired private DataSetService dataSetService; - @Autowired - private ModelService modelService; + @Autowired private ModelService modelService; @Test public void test_getDomainList_alice() { @@ -33,7 +31,8 @@ public class SchemaAuthTest extends BaseTest { setDomainNotOpenToAll(); List domainResps = domainService.getDomainListWithAdminAuth(user); List expectedDomainBizNames = Lists.newArrayList("supersonic", "singer"); - Assertions.assertEquals(expectedDomainBizNames, + Assertions.assertEquals( + expectedDomainBizNames, domainResps.stream().map(DomainResp::getBizName).collect(Collectors.toList())); } @@ -42,16 +41,19 @@ public class SchemaAuthTest extends BaseTest { User user = DataUtils.getUserAlice(); List modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); List expectedModelBizNames = Lists.newArrayList("user_department", "singer"); - Assertions.assertEquals(expectedModelBizNames, + Assertions.assertEquals( + expectedModelBizNames, modelResps.stream().map(ModelResp::getBizName).collect(Collectors.toList())); } @Test public void test_getVisibleModelList_alice() { User user = DataUtils.getUserAlice(); - List modelResps = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE); + List modelResps = + modelService.getModelListWithAuth(user, null, AuthType.VISIBLE); List expectedModelBizNames = Lists.newArrayList("user_department", "singer"); - Assertions.assertEquals(expectedModelBizNames, + Assertions.assertEquals( + expectedModelBizNames, modelResps.stream().map(ModelResp::getBizName).collect(Collectors.toList())); } @@ -60,7 +62,8 @@ public class SchemaAuthTest extends BaseTest { User user = DataUtils.getUserAlice(); List dataSetResps = dataSetService.getDataSetsInheritAuth(user, 0L); List expectedDataSetBizNames = Lists.newArrayList("singer"); - Assertions.assertEquals(expectedDataSetBizNames, + Assertions.assertEquals( + expectedDataSetBizNames, dataSetResps.stream().map(DataSetResp::getBizName).collect(Collectors.toList())); } @@ -69,7 +72,8 @@ public class SchemaAuthTest extends BaseTest { User user = DataUtils.getUserJack(); List domainResps = domainService.getDomainListWithAdminAuth(user); List expectedDomainBizNames = Lists.newArrayList("supersonic"); - Assertions.assertEquals(expectedDomainBizNames, + Assertions.assertEquals( + expectedDomainBizNames, domainResps.stream().map(DomainResp::getBizName).collect(Collectors.toList())); } @@ -77,9 +81,10 @@ public class SchemaAuthTest extends BaseTest { public void test_getModelList_jack() { User user = DataUtils.getUserJack(); List modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); - List expectedModelBizNames = Lists.newArrayList("user_department", - "s2_pv_uv_statis", "s2_stay_time_statis"); - Assertions.assertEquals(expectedModelBizNames, + List expectedModelBizNames = + Lists.newArrayList("user_department", "s2_pv_uv_statis", "s2_stay_time_statis"); + Assertions.assertEquals( + expectedModelBizNames, modelResps.stream().map(ModelResp::getBizName).collect(Collectors.toList())); } @@ -88,8 +93,8 @@ public class SchemaAuthTest extends BaseTest { User user = DataUtils.getUserJack(); List dataSetResps = dataSetService.getDataSetsInheritAuth(user, 0L); List expectedDataSetBizNames = Lists.newArrayList("s2", "singer"); - Assertions.assertEquals(expectedDataSetBizNames, + Assertions.assertEquals( + expectedDataSetBizNames, dataSetResps.stream().map(DataSetResp::getBizName).collect(Collectors.toList())); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagObjectTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagObjectTest.java index 8d091a679..fa3220583 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagObjectTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagObjectTest.java @@ -13,8 +13,7 @@ import java.util.List; public class TagObjectTest extends BaseTest { - @Autowired - private TagObjectService tagObjectService; + @Autowired private TagObjectService tagObjectService; @Test void testCreateTagObject() throws Exception { @@ -32,7 +31,8 @@ public class TagObjectTest extends BaseTest { BeanUtils.copyProperties(tagObjectResp, tagObjectReqUpdate); tagObjectReqUpdate.setName("艺人1"); tagObjectService.update(tagObjectReqUpdate, User.getFakeUser()); - TagObjectResp tagObject = tagObjectService.getTagObject(tagObjectReqUpdate.getId(), User.getFakeUser()); + TagObjectResp tagObject = + tagObjectService.getTagObject(tagObjectReqUpdate.getId(), User.getFakeUser()); tagObjectService.delete(tagObject.getId(), User.getFakeUser(), false); } @@ -53,5 +53,4 @@ public class TagObjectTest extends BaseTest { tagObjectReq.setBizName("new_singer"); return tagObjectReq; } - -} \ No newline at end of file +} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java index 2cf885617..1fec08bd2 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java @@ -13,15 +13,14 @@ import org.springframework.beans.factory.annotation.Autowired; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class TagTest extends BaseTest { - @Autowired - private TagQueryService tagQueryService; + @Autowired private TagQueryService tagQueryService; @Test public void testQueryTagValue() throws Exception { ItemValueReq itemValueReq = new ItemValueReq(); itemValueReq.setId(1L); - ItemValueResp itemValueResp = tagQueryService.queryTagValue(itemValueReq, User.getFakeUser()); + ItemValueResp itemValueResp = + tagQueryService.queryTagValue(itemValueReq, User.getFakeUser()); Assertions.assertNotNull(itemValueResp); } - -} \ No newline at end of file +} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java index b8c4cd361..00ff9d6d1 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp; @@ -18,8 +17,10 @@ public class TranslateTest extends BaseTest { @Test public void testSqlExplain() throws Exception { String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "; - SemanticTranslateResp explain = semanticLayerService.translate(QueryReqBuilder.buildS2SQLReq(sql, - DataUtils.getMetricAgentView()), User.getFakeUser()); + SemanticTranslateResp explain = + semanticLayerService.translate( + QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView()), + User.getFakeUser()); assertNotNull(explain); assertNotNull(explain.getQuerySQL()); assertTrue(explain.getQuerySQL().contains("department")); @@ -29,11 +30,11 @@ public class TranslateTest extends BaseTest { @Test public void testStructExplain() throws Exception { QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department")); - SemanticTranslateResp explain = semanticLayerService.translate(queryStructReq, User.getFakeUser()); + SemanticTranslateResp explain = + semanticLayerService.translate(queryStructReq, User.getFakeUser()); assertNotNull(explain); assertNotNull(explain.getQuerySQL()); assertTrue(explain.getQuerySQL().contains("department")); assertTrue(explain.getQuerySQL().contains("pv")); } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/provider/ModelProviderTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/provider/ModelProviderTest.java index 95b862ee1..7736de550 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/provider/ModelProviderTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/provider/ModelProviderTest.java @@ -51,9 +51,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setEndpoint(QianfanModelFactory.DEFAULT_ENDPOINT); ChatLanguageModel chatModel = ModelProvider.getChatModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - chatModel.generate("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + chatModel.generate("hi"); + }); } @Test @@ -65,9 +67,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setApiKey("e2724491714b3b2a0274e987905f1001.5JyHgf4vbZVJ7gC5"); ChatLanguageModel chatModel = ModelProvider.getChatModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - chatModel.generate("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + chatModel.generate("hi"); + }); } @Test @@ -80,9 +84,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setApiKey(ParameterConfig.DEMO); ChatLanguageModel chatModel = ModelProvider.getChatModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - chatModel.generate("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + chatModel.generate("hi"); + }); } @Test @@ -94,9 +100,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setApiKey(ParameterConfig.DEMO); ChatLanguageModel chatModel = ModelProvider.getChatModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - chatModel.generate("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + chatModel.generate("hi"); + }); } @Test @@ -132,9 +140,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setApiKey(ParameterConfig.DEMO); EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - embeddingModel.embed("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + embeddingModel.embed("hi"); + }); } @Test @@ -146,9 +156,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setApiKey(ParameterConfig.DEMO); EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - embeddingModel.embed("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + embeddingModel.embed("hi"); + }); } @Test @@ -161,9 +173,11 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setSecretKey(ParameterConfig.DEMO); EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - embeddingModel.embed("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + embeddingModel.embed("hi"); + }); } @Test @@ -175,8 +189,10 @@ public class ModelProviderTest extends BaseApplication { modelConfig.setApiKey("e2724491714b3b2a0274e987905f1001.5JyHgf4vbZVJ7gC5"); EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(modelConfig); - assertThrows(RuntimeException.class, () -> { - embeddingModel.embed("hi"); - }); + assertThrows( + RuntimeException.class, + () -> { + embeddingModel.embed("hi"); + }); } } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/AESEncryptionUtilTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/AESEncryptionUtilTest.java index 3af026c7a..dfd05c642 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/AESEncryptionUtilTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/AESEncryptionUtilTest.java @@ -44,6 +44,5 @@ public class AESEncryptionUtilTest { System.out.println("after AES/ECB encrypt：" + encryptStr); String decryptStr = AESEncryptionUtil.aesDecryptECB(encryptStr); System.out.println("after AES/ECB decrypt：" + decryptStr); - } } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 629fb793c..e4c3aa80c 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -61,13 +61,15 @@ public class DataUtils { } public static SchemaElement getSchemaElement(String name) { - return SchemaElement.builder() - .name(name) - .build(); + return SchemaElement.builder().name(name).build(); } - public static QueryFilter getFilter(String bizName, FilterOperatorEnum filterOperatorEnum, - Object value, String name, Long elementId) { + public static QueryFilter getFilter( + String bizName, + FilterOperatorEnum filterOperatorEnum, + Object value, + String name, + Long elementId) { QueryFilter filter = new QueryFilter(); filter.setBizName(bizName); filter.setOperator(filterOperatorEnum); @@ -87,8 +89,12 @@ public class DataUtils { return dateInfo; } - public static DateConf getDateConf(DateConf.DateMode dateMode, Integer unit, - String period, String startDate, String endDate) { + public static DateConf getDateConf( + DateConf.DateMode dateMode, + Integer unit, + String period, + String startDate, + String endDate) { DateConf dateInfo = new DateConf(); dateInfo.setUnit(unit); dateInfo.setDateMode(dateMode); @@ -98,7 +104,8 @@ public class DataUtils { return dateInfo; } - public static DateConf getDateConf(DateConf.DateMode dateMode, String startDate, String endDate) { + public static DateConf getDateConf( + DateConf.DateMode dateMode, String startDate, String endDate) { DateConf dateInfo = new DateConf(); dateInfo.setDateMode(dateMode); dateInfo.setStartDate(startDate); @@ -106,7 +113,8 @@ public class DataUtils { return dateInfo; } - public static DateConf getDateConf(DateConf.DateMode dateMode, String startDate, String endDate, int unit) { + public static DateConf getDateConf( + DateConf.DateMode dateMode, String startDate, String endDate, int unit) { DateConf dateInfo = new DateConf(); dateInfo.setDateMode(dateMode); dateInfo.setStartDate(startDate); @@ -141,8 +149,14 @@ public class DataUtils { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); ruleQueryTool.setDataSetIds(Lists.newArrayList(-1L)); - ruleQueryTool.setQueryModes(Lists.newArrayList("METRIC_ID", "METRIC_FILTER", "METRIC_MODEL", - "TAG_DETAIL", "TAG_LIST_FILTER", "TAG_ID")); + ruleQueryTool.setQueryModes( + Lists.newArrayList( + "METRIC_ID", + "METRIC_FILTER", + "METRIC_MODEL", + "TAG_DETAIL", + "TAG_LIST_FILTER", + "TAG_ID")); return ruleQueryTool; } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/HanlpTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/HanlpTest.java index 276a60099..dbd9efc86 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/HanlpTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/HanlpTest.java @@ -6,11 +6,12 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.chat.knowledge.DatabaseMapResult; import com.tencent.supersonic.headless.chat.knowledge.MapResult; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; -import java.util.ArrayList; -import java.util.List; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; + public class HanlpTest { @Test @@ -36,4 +37,4 @@ public class HanlpTest { HanlpHelper.transLetterOriginal(mapResults); Assert.assertEquals(mapResults.size(), 2); } -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index 12ef73bcb..bcdcc39d0 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,7 @@ 15.0.2 15.0.2 2.10.1 + 2.27.1 @@ -204,12 +205,6 @@ langchain4j-chatglm ${langchain4j.version} - - com.google.code.gson gson @@ -264,8 +259,29 @@ - org.apache.maven.plugins - maven-checkstyle-plugin + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + 1.7 + + + + javax,java,scala,\# + + + + + + spotless-check + validate + + check + + + @@ -275,36 +291,6 @@ sonar-maven-plugin 3.6.0.1398 - - org.apache.maven.plugins - maven-checkstyle-plugin - 3.1.2 - - - com.puppycrawl.tools - checkstyle - 9.3 - - - - checkstyle/checkstyle.xml - UTF-8 - true - true - false - true - - - - checkstyle-validation - validate - true - - check - - - - org.apache.maven.plugins maven-surefire-plugin