(improvement)[build] Use Spotless to customize the code formatting (#1750)

This commit is contained in:
lexluo09
2024-10-04 00:05:04 +08:00
committed by GitHub
parent 44d1cde34f
commit 71a9954be5
521 changed files with 7811 additions and 13046 deletions

View File

@@ -7,4 +7,5 @@ import java.lang.annotation.Target;
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface AuthenticationIgnore {}
public @interface AuthenticationIgnore {
}

View File

@@ -24,8 +24,7 @@ public class AuthenticationConfig {
@Value("${s2.authentication.token.default.appKey:supersonic}")
private String tokenDefaultAppKey;
@Value(
"${s2.authentication.token.appSecret:supersonic:WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk"
@Value("${s2.authentication.token.appSecret:supersonic:WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk"
+ "783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==}")
private String tokenAppSecret;
@@ -48,8 +47,7 @@ public class AuthenticationConfig {
private Long tokenTimeout;
public Map<String, String> getAppKeyToSecretMap() {
return Arrays.stream(this.tokenAppSecret.split(","))
.map(s -> s.split(":"))
return Arrays.stream(this.tokenAppSecret.split(",")).map(s -> s.split(":"))
.collect(Collectors.toMap(e -> e[0].trim(), e -> e[1].trim()));
}
}

View File

@@ -20,8 +20,8 @@ public class User {
private Integer isAdmin;
public static User get(
Long id, String name, String displayName, String email, Integer isAdmin) {
public static User get(Long id, String name, String displayName, String email,
Integer isAdmin) {
return new User(id, name, displayName, email, isAdmin);
}

View File

@@ -9,24 +9,14 @@ public class UserWithPassword extends User {
private String password;
public UserWithPassword(
Long id,
String name,
String displayName,
String email,
String password,
public UserWithPassword(Long id, String name, String displayName, String email, String password,
Integer isAdmin) {
super(id, name, displayName, email, isAdmin);
this.password = password;
}
public static UserWithPassword get(
Long id,
String name,
String displayName,
String email,
String password,
Integer isAdmin) {
public static UserWithPassword get(Long id, String name, String displayName, String email,
String password, Integer isAdmin) {
return new UserWithPassword(id, name, displayName, email, password, isAdmin);
}
}

View File

@@ -12,8 +12,8 @@ import java.util.Set;
public interface UserService {
User getCurrentUser(
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse);
User getCurrentUser(HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse);
List<String> getUserNames();

View File

@@ -52,12 +52,10 @@ public class DefaultUserAdaptor implements UserAdaptor {
new Organization("1", "0", "SuperSonic", "SuperSonic", Lists.newArrayList(), true);
Organization hr =
new Organization("2", "1", "Hr", "SuperSonic/Hr", Lists.newArrayList(), false);
Organization sales =
new Organization(
"3", "1", "Sales", "SuperSonic/Sales", Lists.newArrayList(), false);
Organization marketing =
new Organization(
"4", "1", "Marketing", "SuperSonic/Marketing", Lists.newArrayList(), false);
Organization sales = new Organization("3", "1", "Sales", "SuperSonic/Sales",
Lists.newArrayList(), false);
Organization marketing = new Organization("4", "1", "Marketing", "SuperSonic/Marketing",
Lists.newArrayList(), false);
List<Organization> subOrganization = Lists.newArrayList(hr, sales, marketing);
superSonic.setSubOrganizations(subOrganization);
return Lists.newArrayList(superSonic);
@@ -113,18 +111,11 @@ public class DefaultUserAdaptor implements UserAdaptor {
throw new RuntimeException("user not exist,please register");
}
try {
String password =
AESEncryptionUtil.encrypt(
userReq.getPassword(),
String password = AESEncryptionUtil.encrypt(userReq.getPassword(),
AESEncryptionUtil.getBytesFromString(userDO.getSalt()));
if (userDO.getPassword().equals(password)) {
UserWithPassword user =
UserWithPassword.get(
userDO.getId(),
userDO.getName(),
userDO.getDisplayName(),
userDO.getEmail(),
userDO.getPassword(),
UserWithPassword user = UserWithPassword.get(userDO.getId(), userDO.getName(),
userDO.getDisplayName(), userDO.getEmail(), userDO.getPassword(),
userDO.getIsAdmin());
return user;
} else {

View File

@@ -68,8 +68,8 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor {
try {
if (request instanceof StandardMultipartHttpServletRequest) {
RequestFacade servletRequest =
(RequestFacade)
((StandardMultipartHttpServletRequest) request).getRequest();
(RequestFacade) ((StandardMultipartHttpServletRequest) request)
.getRequest();
Class<? extends HttpServletRequest> servletRequestClazz = servletRequest.getClass();
Field request1 = servletRequestClazz.getDeclaredField("request");
request1.setAccessible(true);

View File

@@ -22,9 +22,8 @@ import java.lang.reflect.Method;
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
@Override
public boolean preHandle(
HttpServletRequest request, HttpServletResponse response, Object handler)
throws AccessException {
public boolean preHandle(HttpServletRequest request, HttpServletResponse response,
Object handler) throws AccessException {
authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class);
userServiceImpl = ContextUtils.getBean(UserServiceImpl.class);
userTokenUtils = ContextUtils.getBean(UserTokenUtils.class);
@@ -74,11 +73,9 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
}
private void setContext(String userName, HttpServletRequest request) {
ThreadContext threadContext =
ThreadContext.builder()
ThreadContext threadContext = ThreadContext.builder()
.token(request.getHeader(authenticationConfig.getTokenHttpHeaderKey()))
.userName(userName)
.build();
.userName(userName).build();
s2ThreadContext.set(threadContext);
}
}

View File

@@ -13,17 +13,14 @@ public class InterceptorFactory implements WebMvcConfigurer {
private List<AuthenticationInterceptor> authenticationInterceptors;
public InterceptorFactory() {
authenticationInterceptors =
SpringFactoriesLoader.loadFactories(
AuthenticationInterceptor.class,
Thread.currentThread().getContextClassLoader());
authenticationInterceptors = SpringFactoriesLoader.loadFactories(
AuthenticationInterceptor.class, Thread.currentThread().getContextClassLoader());
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
for (AuthenticationInterceptor authenticationInterceptor : authenticationInterceptors) {
registry.addInterceptor(authenticationInterceptor)
.addPathPatterns("/**")
registry.addInterceptor(authenticationInterceptor).addPathPatterns("/**")
.excludePathPatterns("/", "/webapp/**", "/error");
}
}

View File

@@ -138,8 +138,8 @@ public class UserDOExample {
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(
String condition, Object value1, Object value2, String property) {
protected void addCriterion(String condition, Object value1, Object value2,
String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
@@ -628,8 +628,8 @@ public class UserDOExample {
this(condition, value, null);
}
protected Criterion(
String condition, Object value, Object secondValue, String typeHandler) {
protected Criterion(String condition, Object value, Object secondValue,
String typeHandler) {
super();
this.condition = condition;
this.value = value;

View File

@@ -30,8 +30,8 @@ public class UserController {
}
@GetMapping("/getCurrentUser")
public User getCurrentUser(
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
public User getCurrentUser(HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
return userService.getCurrentUser(httpServletRequest, httpServletResponse);
}

View File

@@ -27,8 +27,8 @@ public class UserServiceImpl implements UserService {
}
@Override
public User getCurrentUser(
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
public User getCurrentUser(HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
if (user != null) {
SystemConfig systemConfig = sysParameterService.getSystemConfig();

View File

@@ -18,8 +18,8 @@ public class UserStrategyFactory {
private AuthenticationConfig authenticationConfig;
public UserStrategyFactory(
AuthenticationConfig authenticationConfig, List<UserStrategy> userStrategyList) {
public UserStrategyFactory(AuthenticationConfig authenticationConfig,
List<UserStrategy> userStrategyList) {
this.authenticationConfig = authenticationConfig;
this.userStrategyList = userStrategyList;
}

View File

@@ -17,8 +17,7 @@ public class ComponentFactory {
}
private static <T> T init(Class<T> factoryType) {
return SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader())
.get(0);
return SpringFactoriesLoader
.loadFactories(factoryType, Thread.currentThread().getContextClassLoader()).get(0);
}
}

View File

@@ -48,8 +48,7 @@ public class UserTokenUtils {
Map<String, Object> claims = new HashMap<>(5);
claims.put(TOKEN_USER_ID, user.getId());
claims.put(TOKEN_USER_NAME, StringUtils.isEmpty(user.getName()) ? "" : user.getName());
claims.put(
TOKEN_USER_PASSWORD,
claims.put(TOKEN_USER_PASSWORD,
StringUtils.isEmpty(user.getPassword()) ? "" : user.getPassword());
claims.put(TOKEN_USER_DISPLAY_NAME, user.getDisplayName());
claims.put(TOKEN_CREATE_TIME, System.currentTimeMillis());
@@ -83,9 +82,7 @@ public class UserTokenUtils {
String userName = String.valueOf(claims.get(TOKEN_USER_NAME));
String email = String.valueOf(claims.get(TOKEN_USER_EMAIL));
String displayName = String.valueOf(claims.get(TOKEN_USER_DISPLAY_NAME));
Integer isAdmin =
claims.get(TOKEN_IS_ADMIN) == null
? 0
Integer isAdmin = claims.get(TOKEN_IS_ADMIN) == null ? 0
: Integer.parseInt(claims.get(TOKEN_IS_ADMIN).toString());
return User.get(userId, userName, displayName, email, isAdmin);
}
@@ -105,9 +102,7 @@ public class UserTokenUtils {
String email = String.valueOf(claims.get(TOKEN_USER_EMAIL));
String displayName = String.valueOf(claims.get(TOKEN_USER_DISPLAY_NAME));
String password = String.valueOf(claims.get(TOKEN_USER_PASSWORD));
Integer isAdmin =
claims.get(TOKEN_IS_ADMIN) == null
? 0
Integer isAdmin = claims.get(TOKEN_IS_ADMIN) == null ? 0
: Integer.parseInt(claims.get(TOKEN_IS_ADMIN).toString());
return UserWithPassword.get(userId, userName, displayName, email, password, isAdmin);
}
@@ -121,11 +116,8 @@ public class UserTokenUtils {
try {
String tokenSecret = getTokenSecret(appKey);
Claims claims =
Jwts.parser()
.setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8))
.build()
.parseClaimsJws(getTokenString(token))
.getBody();
Jwts.parser().setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8))
.build().parseClaimsJws(getTokenString(token)).getBody();
return Optional.of(claims);
} catch (Exception e) {
log.info("can not getClaims from appKey:{} token:{}, please login", appKey, token);
@@ -149,15 +141,10 @@ public class UserTokenUtils {
Date expirationDate = new Date(expiration);
String tokenSecret = getTokenSecret(appKey);
return Jwts.builder()
.setClaims(claims)
.setSubject(claims.get(TOKEN_USER_NAME).toString())
return Jwts.builder().setClaims(claims).setSubject(claims.get(TOKEN_USER_NAME).toString())
.setExpiration(expirationDate)
.signWith(
new SecretKeySpec(
tokenSecret.getBytes(StandardCharsets.UTF_8),
SignatureAlgorithm.HS512.getJcaName()),
SignatureAlgorithm.HS512)
.signWith(new SecretKeySpec(tokenSecret.getBytes(StandardCharsets.UTF_8),
SignatureAlgorithm.HS512.getJcaName()), SignatureAlgorithm.HS512)
.compact();
}

View File

@@ -31,8 +31,7 @@ public class AuthController {
}
@GetMapping("/queryGroup")
public List<AuthGroup> queryAuthGroup(
@RequestParam("modelId") String modelId,
public List<AuthGroup> queryAuthGroup(@RequestParam("modelId") String modelId,
@RequestParam(value = "groupId", required = false) Integer groupId) {
return authService.queryAuthGroups(modelId, groupId);
}
@@ -69,10 +68,8 @@ public class AuthController {
* @return
*/
@PostMapping("/queryAuthorizedRes")
public AuthorizedResourceResp queryAuthorizedResources(
@RequestBody QueryAuthResReq req,
HttpServletRequest request,
HttpServletResponse response) {
public AuthorizedResourceResp queryAuthorizedResources(@RequestBody QueryAuthResReq req,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return authService.queryAuthorizedResources(req, user);
}

View File

@@ -39,17 +39,14 @@ public class AuthServiceImpl implements AuthService {
List<String> rows =
jdbcTemplate.queryForList("select config from s2_auth_groups", String.class);
Gson g = new Gson();
return rows.stream()
.map(row -> g.fromJson(row, AuthGroup.class))
return rows.stream().map(row -> g.fromJson(row, AuthGroup.class))
.collect(Collectors.toList());
}
@Override
public List<AuthGroup> queryAuthGroups(String modelId, Integer groupId) {
return load().stream()
.filter(
group ->
(Objects.isNull(groupId) || groupId.equals(group.getGroupId()))
.filter(group -> (Objects.isNull(groupId) || groupId.equals(group.getGroupId()))
&& modelId.equals(group.getModelId().toString()))
.collect(Collectors.toList());
}
@@ -65,15 +62,11 @@ public class AuthServiceImpl implements AuthService {
nextGroupId = obj + 1;
}
group.setGroupId(nextGroupId);
jdbcTemplate.update(
"insert into s2_auth_groups (group_id, config) values (?, ?);",
nextGroupId,
g.toJson(group));
jdbcTemplate.update("insert into s2_auth_groups (group_id, config) values (?, ?);",
nextGroupId, g.toJson(group));
} else {
jdbcTemplate.update(
"update s2_auth_groups set config = ? where group_id = ?;",
g.toJson(group),
group.getGroupId());
jdbcTemplate.update("update s2_auth_groups set config = ? where group_id = ?;",
g.toJson(group), group.getGroupId());
}
}
@@ -119,12 +112,9 @@ public class AuthServiceImpl implements AuthService {
return resource;
}
private List<AuthGroup> getAuthGroups(
List<Long> modelIds, String userName, List<String> departmentIds) {
List<AuthGroup> groups =
load().stream()
.filter(
group -> {
private List<AuthGroup> getAuthGroups(List<Long> modelIds, String userName,
List<String> departmentIds) {
List<AuthGroup> groups = load().stream().filter(group -> {
if (!modelIds.contains(group.getModelId())) {
return false;
}
@@ -133,16 +123,13 @@ public class AuthServiceImpl implements AuthService {
return true;
}
for (String departmentId : departmentIds) {
if (!CollectionUtils.isEmpty(
group.getAuthorizedDepartmentIds())
&& group.getAuthorizedDepartmentIds()
.contains(departmentId)) {
if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds())
&& group.getAuthorizedDepartmentIds().contains(departmentId)) {
return true;
}
}
return false;
})
.collect(Collectors.toList());
}).collect(Collectors.toList());
log.info("user:{} department:{} authGroups:{}", userName, departmentIds, groups);
return groups;
}

View File

@@ -4,8 +4,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import org.apache.commons.lang3.StringUtils;
public enum MemoryReviewResult {
POSITIVE,
NEGATIVE;
POSITIVE, NEGATIVE;
public static MemoryReviewResult getMemoryReviewResult(String value) {
String validValue = StringUtils.trim(value);

View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.chat.api.pojo.enums;
public enum MemoryStatus {
PENDING,
ENABLED,
DISABLED;
PENDING, ENABLED, DISABLED;
}

View File

@@ -14,7 +14,8 @@ public class KnowledgeInfoReq {
private String bizName;
/** type: IntentionTypeEnum temporarily only supports dimension-related information */
@NotNull private TypeEnums type = TypeEnums.DIMENSION;
@NotNull
private TypeEnums type = TypeEnums.DIMENSION;
private Boolean searchEnable = false;

View File

@@ -43,16 +43,12 @@ public class Agent extends RecordInfo {
return Lists.newArrayList();
}
List<Map> toolList = (List) map.get("tools");
return toolList.stream()
.filter(
tool -> {
return toolList.stream().filter(tool -> {
if (Objects.isNull(type)) {
return true;
}
return type.name().equals(tool.get("type"));
})
.map(JSONObject::toJSONString)
.collect(Collectors.toList());
}).map(JSONObject::toJSONString).collect(Collectors.toList());
}
public boolean enableSearch() {
@@ -72,8 +68,7 @@ public class Agent extends RecordInfo {
if (CollectionUtils.isEmpty(tools)) {
return Lists.newArrayList();
}
return tools.stream()
.map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class))
return tools.stream().map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class))
.collect(Collectors.toList());
}
@@ -120,10 +115,8 @@ public class Agent extends RecordInfo {
if (CollectionUtils.isEmpty(commonAgentTools)) {
return new HashSet<>();
}
return commonAgentTools.stream()
.map(NL2SQLTool::getDataSetIds)
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds))
.flatMap(Collection::stream)
return commonAgentTools.stream().map(NL2SQLTool::getDataSetIds)
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds)).flatMap(Collection::stream)
.collect(Collectors.toSet());
}
}

View File

@@ -4,9 +4,7 @@ import java.util.HashMap;
import java.util.Map;
public enum AgentToolType {
NL2SQL_RULE("基于规则Text-to-SQL"),
NL2SQL_LLM("基于大模型Text-to-SQL"),
PLUGIN("第三方插件");
NL2SQL_RULE("基于规则Text-to-SQL"), NL2SQL_LLM("基于大模型Text-to-SQL"), PLUGIN("第三方插件");
private String title;

View File

@@ -26,14 +26,10 @@ import static com.tencent.supersonic.chat.server.parser.ParserConfig.PARSER_MULT
public class PlainTextExecutor implements ChatQueryExecutor {
private static final String INSTRUCTION =
""
+ "#Role: You are a nice person to talk to.\n"
private static final String INSTRUCTION = "" + "#Role: You are a nice person to talk to.\n"
+ "#Task: Respond quickly and nicely to the user."
+ "#Rules: 1.ALWAYS use the same language as the input.\n"
+ "#History Inputs: %s\n"
+ "#Current Input: %s\n"
+ "#Your response: ";
+ "#Rules: 1.ALWAYS use the same language as the input.\n" + "#History Inputs: %s\n"
+ "#Current Input: %s\n" + "#Your response: ";
@Override
public QueryResult execute(ExecuteContext executeContext) {
@@ -41,10 +37,7 @@ public class PlainTextExecutor implements ChatQueryExecutor {
return null;
}
String promptStr =
String.format(
INSTRUCTION,
getHistoryInputs(executeContext),
String promptStr = String.format(INSTRUCTION, getHistoryInputs(executeContext),
executeContext.getQueryText());
Prompt prompt = PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP);
@@ -74,15 +67,12 @@ public class PlainTextExecutor implements ChatQueryExecutor {
Boolean globalMultiTurnConfig =
Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE));
Boolean multiTurnConfig =
agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn()
agentMultiTurnConfig != null ? agentMultiTurnConfig.isEnableMultiTurn()
: globalMultiTurnConfig;
if (Boolean.TRUE.equals(multiTurnConfig)) {
List<QueryResp> queryResps = getHistoryQueries(executeContext.getChatId(), 5);
queryResps.stream()
.forEach(
p -> {
queryResps.stream().forEach(p -> {
historyInput.append(p.getQueryText());
historyInput.append(";");
});
@@ -93,18 +83,13 @@ public class PlainTextExecutor implements ChatQueryExecutor {
private List<QueryResp> getHistoryQueries(int chatId, int multiNum) {
ChatManageService chatManageService = ContextUtils.getBean(ChatManageService.class);
List<QueryResp> contextualParseInfoList =
chatManageService.getChatQueries(chatId).stream()
.filter(
q ->
Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState()
== QueryState.SUCCESS)
List<QueryResp> contextualParseInfoList = chatManageService.getChatQueries(chatId).stream()
.filter(q -> Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState() == QueryState.SUCCESS)
.collect(Collectors.toList());
List<QueryResp> contextualList =
contextualParseInfoList.subList(
0, Math.min(multiNum, contextualParseInfoList.size()));
List<QueryResp> contextualList = contextualParseInfoList.subList(0,
Math.min(multiNum, contextualParseInfoList.size()));
Collections.reverse(contextualList);
return contextualList;

View File

@@ -31,34 +31,25 @@ public class SqlExecutor implements ChatQueryExecutor {
QueryResult queryResult = doExecute(executeContext);
if (queryResult != null) {
String textResult =
ResultFormatter.transform2TextNew(
queryResult.getQueryColumns(), queryResult.getQueryResults());
String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
queryResult.getQueryResults());
queryResult.setTextResult(textResult);
if (queryResult.getQueryState().equals(QueryState.SUCCESS)
&& queryResult.getQueryMode().equals(LLMSqlQuery.QUERY_MODE)) {
Text2SQLExemplar exemplar =
JsonUtil.toObject(
JsonUtil.toString(
executeContext
.getParseInfo()
.getProperties()
JsonUtil.toString(executeContext.getParseInfo().getProperties()
.get(Text2SQLExemplar.PROPERTY_KEY)),
Text2SQLExemplar.class);
MemoryService memoryService = ContextUtils.getBean(MemoryService.class);
memoryService.createMemory(
ChatMemoryDO.builder()
.agentId(executeContext.getAgent().getId())
.status(MemoryStatus.PENDING)
.question(exemplar.getQuestion())
.sideInfo(exemplar.getSideInfo())
.dbSchema(exemplar.getDbSchema())
.s2sql(exemplar.getSql())
memoryService.createMemory(ChatMemoryDO.builder()
.agentId(executeContext.getAgent().getId()).status(MemoryStatus.PENDING)
.question(exemplar.getQuestion()).sideInfo(exemplar.getSideInfo())
.dbSchema(exemplar.getDbSchema()).s2sql(exemplar.getSql())
.createdBy(executeContext.getUser().getName())
.updatedBy(executeContext.getUser().getName())
.createdAt(new Date())
.updatedBy(executeContext.getUser().getName()).createdAt(new Date())
.build());
}
}

View File

@@ -27,25 +27,22 @@ public class MemoryReviewTask {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private static final String INSTRUCTION =
""
private static final String INSTRUCTION = ""
+ "\n#Role: You are a senior data engineer experienced in writing SQL."
+ "\n#Task: Your will be provided with a user question and the SQL written by junior engineer,"
+ "please take a review and give your opinion."
+ "\n#Rules: "
+ "please take a review and give your opinion." + "\n#Rules: "
+ "1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),comment=(your comment)`."
+ "2.NO NEED to include date filter in the where clause if not explicitly expressed in the `Question`."
+ "\n#Question: %s"
+ "\n#Schema: %s"
+ "\n#SideInfo: %s"
+ "\n#SQL: %s"
+ "\n#Question: %s" + "\n#Schema: %s" + "\n#SideInfo: %s" + "\n#SQL: %s"
+ "\n#Response: ";
private static final Pattern OUTPUT_PATTERN = Pattern.compile("opinion=(.*),.*comment=(.*)");
@Autowired private MemoryService memoryService;
@Autowired
private MemoryService memoryService;
@Autowired private AgentService agentService;
@Autowired
private AgentService agentService;
@Scheduled(fixedDelay = 60 * 1000)
public void review() {
@@ -78,8 +75,8 @@ public class MemoryReviewTask {
}
private String createPromptString(ChatMemoryDO m) {
return String.format(
INSTRUCTION, m.getQuestion(), m.getDbSchema(), m.getSideInfo(), m.getS2sql());
return String.format(INSTRUCTION, m.getQuestion(), m.getDbSchema(), m.getSideInfo(),
m.getS2sql());
}
private void processResponse(String response, ChatMemoryDO m) {

View File

@@ -21,12 +21,9 @@ public class NL2PluginParser implements ChatQueryParser {
return;
}
pluginRecognizers.forEach(
pluginRecognizer -> {
pluginRecognizers.forEach(pluginRecognizer -> {
pluginRecognizer.recognize(parseContext, parseResp);
log.info(
"{} recallResult:{}",
pluginRecognizer.getClass().getSimpleName(),
log.info("{} recallResult:{}", pluginRecognizer.getClass().getSimpleName(),
JsonUtil.toString(parseResp));
});
}

View File

@@ -52,33 +52,27 @@ public class NL2SQLParser implements ChatQueryParser {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private static final String REWRITE_USER_QUESTION_INSTRUCTION =
""
private static final String REWRITE_USER_QUESTION_INSTRUCTION = ""
+ "#Role: You are a data product manager experienced in data requirements."
+ "#Task: Your will be provided with current and history questions asked by a user,"
+ "along with their mapped schema elements(metric, dimension and value),"
+ "please try understanding the semantics and rewrite a question."
+ "#Rules: "
+ "please try understanding the semantics and rewrite a question." + "#Rules: "
+ "1.ALWAYS keep relevant entities, metrics, dimensions, values and date ranges."
+ "2.ONLY respond with the rewritten question."
+ "#Current Question: {{current_question}}"
+ "#Current Mapped Schema: {{current_schema}}"
+ "#History Question: {{history_question}}"
+ "#History Mapped Schema: {{history_schema}}"
+ "#History SQL: {{history_sql}}"
+ "#History Mapped Schema: {{history_schema}}" + "#History SQL: {{history_sql}}"
+ "#Rewritten Question: ";
private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION =
""
private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION = ""
+ "#Role: You are a data business partner who closely interacts with business people.\n"
+ "#Task: Your will be provided with user input, system output and some examples, "
+ "please respond shortly to teach user how to ask the right question, "
+ "by using `Examples` as references."
+ "#Rules: ALWAYS respond with the same language as the `Input`.\n"
+ "#Input: {{user_question}}\n"
+ "#Output: {{system_message}}\n"
+ "#Examples: {{examples}}\n"
+ "#Response: ";
+ "#Input: {{user_question}}\n" + "#Output: {{system_message}}\n"
+ "#Examples: {{examples}}\n" + "#Response: ";
@Override
public void parse(ParseContext parseContext, ParseResp parseResp) {
@@ -100,11 +94,8 @@ public class NL2SQLParser implements ChatQueryParser {
parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses());
} else {
if (parseContext.enbaleLLM()) {
parseResp.setErrorMsg(
rewriteErrorMessage(
parseContext.getQueryText(),
text2SqlParseResp.getErrorMsg(),
queryNLReq.getDynamicExemplars(),
parseResp.setErrorMsg(rewriteErrorMessage(parseContext.getQueryText(),
text2SqlParseResp.getErrorMsg(), queryNLReq.getDynamicExemplars(),
parseContext.getAgent().getExamples(),
parseContext.getAgent().getModelConfig()));
}
@@ -141,40 +132,26 @@ public class NL2SQLParser implements ChatQueryParser {
StringBuilder textBuilder = new StringBuilder();
textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
metric.ifPresent(
schemaElement ->
textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
List<String> dimensionNames =
parseInfo.getDimensions().stream()
.map(SchemaElement::getName)
.filter(Objects::nonNull)
.collect(Collectors.toList());
metric.ifPresent(schemaElement -> textBuilder.append("**指标:** ")
.append(schemaElement.getName()).append(" "));
List<String> dimensionNames = parseInfo.getDimensions().stream().map(SchemaElement::getName)
.filter(Objects::nonNull).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionNames)) {
textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
}
textBuilder.append("\n\n**筛选条件:** \n");
if (parseInfo.getDateInfo() != null) {
textBuilder
.append("**数据时间:** ")
.append(parseInfo.getDateInfo().getStartDate())
.append("~")
.append(parseInfo.getDateInfo().getEndDate())
.append(" ");
textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate())
.append("~").append(parseInfo.getDateInfo().getEndDate()).append(" ");
}
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
|| CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
queryFilters.addAll(parseInfo.getMetricFilters());
for (QueryFilter queryFilter : queryFilters) {
textBuilder
.append("**")
.append(queryFilter.getName())
.append("**")
.append(" ")
.append(queryFilter.getOperator().getValue())
.append(" ")
.append(queryFilter.getValue())
.append(" ");
textBuilder.append("**").append(queryFilter.getName()).append("**").append(" ")
.append(queryFilter.getOperator().getValue()).append(" ")
.append(queryFilter.getValue()).append(" ");
}
}
parseInfo.setTextInfo(textBuilder.toString());
@@ -187,8 +164,7 @@ public class NL2SQLParser implements ChatQueryParser {
Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE));
Boolean multiTurnConfig =
agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn()
agentMultiTurnConfig != null ? agentMultiTurnConfig.isEnableMultiTurn()
: globalMultiTurnConfig;
if (!Boolean.TRUE.equals(multiTurnConfig)) {
return;
@@ -232,30 +208,20 @@ public class NL2SQLParser implements ChatQueryParser {
QueryNLReq rewrittenQueryNLReq = QueryReqConverter.buildText2SqlQueryReq(parseContext);
MapResp rewrittenQueryMapResult = chatLayerService.performMapping(rewrittenQueryNLReq);
parseContext.setMapInfo(rewrittenQueryMapResult.getMapInfo());
log.info(
"Last Query: {} Current Query: {}, Rewritten Query: {}",
lastQuery.getQueryText(),
currentMapResult.getQueryText(),
rewrittenQuery);
log.info("Last Query: {} Current Query: {}, Rewritten Query: {}", lastQuery.getQueryText(),
currentMapResult.getQueryText(), rewrittenQuery);
}
private String rewriteErrorMessage(
String userQuestion,
String errMsg,
List<Text2SQLExemplar> similarExemplars,
List<String> agentExamples,
private String rewriteErrorMessage(String userQuestion, String errMsg,
List<Text2SQLExemplar> similarExemplars, List<String> agentExamples,
ChatModelConfig modelConfig) {
Map<String, Object> variables = new HashMap<>();
variables.put("user_question", userQuestion);
variables.put("system_message", errMsg);
StringBuilder exampleStr = new StringBuilder();
similarExemplars.forEach(
e ->
exampleStr.append(
String.format(
"<Question:{%s},Schema:{%s}> ",
e.getQuestion(), e.getDbSchema())));
similarExemplars.forEach(e -> exampleStr.append(
String.format("<Question:{%s},Schema:{%s}> ", e.getQuestion(), e.getDbSchema())));
agentExamples.forEach(e -> exampleStr.append(String.format("<Question:{%s}> ", e)));
variables.put("examples", exampleStr);
@@ -297,18 +263,13 @@ public class NL2SQLParser implements ChatQueryParser {
private List<QueryResp> getHistoryQueries(int chatId, int multiNum) {
ChatManageService chatManageService = ContextUtils.getBean(ChatManageService.class);
List<QueryResp> contextualParseInfoList =
chatManageService.getChatQueries(chatId).stream()
.filter(
q ->
Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState()
== QueryState.SUCCESS)
List<QueryResp> contextualParseInfoList = chatManageService.getChatQueries(chatId).stream()
.filter(q -> Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState() == QueryState.SUCCESS)
.collect(Collectors.toList());
List<QueryResp> contextualList =
contextualParseInfoList.subList(
0, Math.min(multiNum, contextualParseInfoList.size()));
List<QueryResp> contextualList = contextualParseInfoList.subList(0,
Math.min(multiNum, contextualParseInfoList.size()));
Collections.reverse(contextualList);
return contextualList;
}
@@ -320,9 +281,8 @@ public class NL2SQLParser implements ChatQueryParser {
ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
int exemplarRecallNumber =
Integer.valueOf(parserConfig.getParameterValue(PARSER_EXEMPLAR_RECALL_NUMBER));
List<Text2SQLExemplar> exemplars =
exemplarManager.recallExemplars(
memoryCollectionName, queryNLReq.getQueryText(), exemplarRecallNumber);
List<Text2SQLExemplar> exemplars = exemplarManager.recallExemplars(memoryCollectionName,
queryNLReq.getQueryText(), exemplarRecallNumber);
queryNLReq.getDynamicExemplars().addAll(exemplars);
}
}

View File

@@ -10,11 +10,6 @@ import org.springframework.stereotype.Service;
public class ParserConfig extends ParameterConfig {
public static final Parameter PARSER_MULTI_TURN_ENABLE =
new Parameter(
"s2.parser.multi-turn.enable",
"false",
"是否开启多轮对话",
"开启多轮对话将消耗更多token",
"bool",
"Parser相关配置");
new Parameter("s2.parser.multi-turn.enable", "false", "是否开启多轮对话", "开启多轮对话将消耗更多token",
"bool", "Parser相关配置");
}

View File

@@ -1,10 +1,7 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
public enum CostType {
MAPPER(1, "mapper"),
PARSER(2, "parser"),
QUERY(3, "query"),
PROCESSOR(4, "processor");
MAPPER(1, "mapper"), PARSER(2, "parser"), QUERY(3, "query"), PROCESSOR(4, "processor");
private Integer type;
private String name;

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface AgentDOMapper extends BaseMapper<AgentDO> {}
public interface AgentDOMapper extends BaseMapper<AgentDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ChatMemoryMapper extends BaseMapper<ChatMemoryDO> {}
public interface ChatMemoryMapper extends BaseMapper<ChatMemoryDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ChatQueryDOMapper extends BaseMapper<ChatQueryDO> {}
public interface ChatQueryDOMapper extends BaseMapper<ChatQueryDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface PluginDOMapper extends BaseMapper<PluginDO> {}
public interface PluginDOMapper extends BaseMapper<PluginDO> {
}

View File

@@ -30,9 +30,7 @@ public interface ChatQueryRepository {
Long createChatQuery(ChatParseReq chatParseReq);
List<ChatParseDO> batchSaveParseInfo(
ChatParseReq chatParseReq,
ParseResp parseResult,
List<ChatParseDO> batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult,
List<SemanticParseInfo> candidateParses);
ChatParseDO getParseInfo(Long questionId, int parseId);

View File

@@ -23,8 +23,8 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
private final ChatConfigHelper chatConfigHelper;
private final ChatConfigMapper chatConfigMapper;
public ChatConfigRepositoryImpl(
ChatConfigHelper chatConfigHelper, ChatConfigMapper chatConfigMapper) {
public ChatConfigRepositoryImpl(ChatConfigHelper chatConfigHelper,
ChatConfigMapper chatConfigMapper) {
this.chatConfigHelper = chatConfigHelper;
this.chatConfigMapper = chatConfigMapper;
}
@@ -52,11 +52,8 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
List<ChatConfigDO> chaConfigDOList = chatConfigMapper.search(filterInternal);
if (!CollectionUtils.isEmpty(chaConfigDOList)) {
chaConfigDOList.stream()
.forEach(
chaConfigDO ->
chaConfigDescriptorList.add(
chatConfigHelper.chatConfigDO2Descriptor(
chaConfigDO.getModelId(), chaConfigDO)));
.forEach(chaConfigDO -> chaConfigDescriptorList.add(chatConfigHelper
.chatConfigDO2Descriptor(chaConfigDO.getModelId(), chaConfigDO)));
}
return chaConfigDescriptorList;
}

View File

@@ -40,11 +40,14 @@ import java.util.stream.Collectors;
@Slf4j
public class ChatQueryRepositoryImpl implements ChatQueryRepository {
@Autowired private ChatQueryDOMapper chatQueryDOMapper;
@Autowired
private ChatQueryDOMapper chatQueryDOMapper;
@Autowired private ChatParseMapper chatParseMapper;
@Autowired
private ChatParseMapper chatParseMapper;
@Autowired private ShowCaseCustomMapper showCaseCustomMapper;
@Autowired
private ShowCaseCustomMapper showCaseCustomMapper;
@Override
public PageInfo<QueryResp> getChatQuery(PageQueryInfoReq pageQueryInfoReq, Long chatId) {
@@ -67,11 +70,9 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
.doSelectPageInfo(() -> chatQueryDOMapper.selectList(queryWrapper));
PageInfo<QueryResp> chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo);
chatQueryVOPageInfo.setList(
pageInfo.getList().stream()
chatQueryVOPageInfo.setList(pageInfo.getList().stream()
.sorted(Comparator.comparingInt(o -> o.getQuestionId().intValue()))
.map(this::convertTo)
.collect(Collectors.toList()));
.map(this::convertTo).collect(Collectors.toList()));
return chatQueryVOPageInfo;
}
@@ -94,22 +95,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
QueryWrapper<ChatQueryDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(ChatQueryDO::getChatId, chatId);
queryWrapper.lambda().orderByDesc(ChatQueryDO::getQuestionId);
return chatQueryDOMapper.selectList(queryWrapper).stream()
.map(q -> convertTo(q))
return chatQueryDOMapper.selectList(queryWrapper).stream().map(q -> convertTo(q))
.collect(Collectors.toList());
}
@Override
public List<QueryResp> queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId) {
return showCaseCustomMapper
.queryShowCase(
pageQueryInfoReq.getLimitStart(),
pageQueryInfoReq.getPageSize(),
agentId,
pageQueryInfoReq.getUserName())
.stream()
.map(this::convertTo)
.collect(Collectors.toList());
.queryShowCase(pageQueryInfoReq.getLimitStart(), pageQueryInfoReq.getPageSize(),
agentId, pageQueryInfoReq.getUserName())
.stream().map(this::convertTo).collect(Collectors.toList());
}
private QueryResp convertTo(ChatQueryDO chatQueryDO) {
@@ -121,9 +116,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
queryResult.setQueryId(chatQueryDO.getQuestionId());
queryResp.setQueryResult(queryResult);
}
queryResp.setSimilarQueries(
JSONObject.parseArray(
chatQueryDO.getSimilarQueries(), SimilarQueryRecallResp.class));
queryResp.setSimilarQueries(JSONObject.parseArray(chatQueryDO.getSimilarQueries(),
SimilarQueryRecallResp.class));
queryResp.setParseTimeCost(
JsonUtil.toObject(chatQueryDO.getParseTimeCost(), ParseTimeCostResp.class));
return queryResp;
@@ -147,9 +141,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
}
@Override
public List<ChatParseDO> batchSaveParseInfo(
ChatParseReq chatParseReq,
ParseResp parseResult,
public List<ChatParseDO> batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult,
List<SemanticParseInfo> candidateParses) {
List<ChatParseDO> chatParseDOList = new ArrayList<>();
getChatParseDO(chatParseReq, parseResult.getQueryId(), candidateParses, chatParseDOList);
@@ -159,11 +151,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return chatParseDOList;
}
public void getChatParseDO(
ChatParseReq chatParseReq,
Long queryId,
List<SemanticParseInfo> parses,
List<ChatParseDO> chatParseDOList) {
public void getChatParseDO(ChatParseReq chatParseReq, Long queryId,
List<SemanticParseInfo> parses, List<ChatParseDO> chatParseDOList) {
for (int i = 0; i < parses.size(); i++) {
ChatParseDO chatParseDO = new ChatParseDO();
chatParseDO.setChatId(chatParseReq.getChatId());

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.plugin;
public enum ParseMode {
EMBEDDING_RECALL,
FUNCTION_CALL;
EMBEDDING_RECALL, FUNCTION_CALL;
}

View File

@@ -46,9 +46,11 @@ import java.util.stream.Collectors;
@Component
public class PluginManager {
@Autowired private EmbeddingConfig embeddingConfig;
@Autowired
private EmbeddingConfig embeddingConfig;
@Autowired private EmbeddingService embeddingService;
@Autowired
private EmbeddingService embeddingService;
public static List<ChatPlugin> getPluginAgentCanSupport(ParseContext parseContext) {
PluginService pluginService = ContextUtils.getBean(PluginService.class);
@@ -57,21 +59,14 @@ public class PluginManager {
if (Objects.isNull(agent)) {
return plugins;
}
List<Long> pluginIds =
getPluginTools(agent).stream()
.map(PluginTool::getPlugins)
.flatMap(Collection::stream)
.collect(Collectors.toList());
List<Long> pluginIds = getPluginTools(agent).stream().map(PluginTool::getPlugins)
.flatMap(Collection::stream).collect(Collectors.toList());
if (CollectionUtils.isEmpty(pluginIds)) {
return Lists.newArrayList();
}
plugins =
plugins.stream()
.filter(plugin -> pluginIds.contains(plugin.getId()))
plugins = plugins.stream().filter(plugin -> pluginIds.contains(plugin.getId()))
.collect(Collectors.toList());
log.info(
"plugins witch can be supported by cur agent :{} {}",
agent.getName(),
log.info("plugins witch can be supported by cur agent :{} {}", agent.getName(),
plugins.stream().map(ChatPlugin::getName).collect(Collectors.toList()));
return plugins;
}
@@ -84,8 +79,7 @@ public class PluginManager {
if (CollectionUtils.isEmpty(tools)) {
return Lists.newArrayList();
}
return tools.stream()
.map(tool -> JSONObject.parseObject(tool, PluginTool.class))
return tools.stream().map(tool -> JSONObject.parseObject(tool, PluginTool.class))
.collect(Collectors.toList());
}
@@ -142,23 +136,18 @@ public class PluginManager {
public RetrieveQueryResult recognize(String embeddingText) {
RetrieveQuery retrieveQuery =
RetrieveQuery.builder()
.queryTextsList(Collections.singletonList(embeddingText))
.build();
RetrieveQuery retrieveQuery = RetrieveQuery.builder()
.queryTextsList(Collections.singletonList(embeddingText)).build();
List<RetrieveQueryResult> resultList =
embeddingService.retrieveQuery(
embeddingConfig.getPresetCollection(),
retrieveQuery,
embeddingConfig.getNResult());
List<RetrieveQueryResult> resultList = embeddingService.retrieveQuery(
embeddingConfig.getPresetCollection(), retrieveQuery, embeddingConfig.getNResult());
if (CollectionUtils.isNotEmpty(resultList)) {
for (RetrieveQueryResult embeddingResp : resultList) {
List<Retrieval> embeddingRetrievals = embeddingResp.getRetrieval();
for (Retrieval embeddingRetrieval : embeddingRetrievals) {
embeddingRetrieval.setId(
getPluginIdFromEmbeddingId(embeddingRetrieval.getId()));
embeddingRetrieval
.setId(getPluginIdFromEmbeddingId(embeddingRetrieval.getId()));
}
}
return resultList.get(0);
@@ -173,8 +162,8 @@ public class PluginManager {
int num = 0;
for (String pattern : exampleQuestions) {
TextSegment query = TextSegment.from(pattern);
TextSegmentConvert.addQueryId(
query, generateUniqueEmbeddingId(num, plugin.getId()));
TextSegmentConvert.addQueryId(query,
generateUniqueEmbeddingId(num, plugin.getId()));
queries.add(query);
num++;
}
@@ -250,14 +239,10 @@ public class PluginManager {
return Sets.newHashSet();
}
return schemaElementMatches.stream()
.filter(
schemaElementMatch ->
SchemaElementType.VALUE.equals(
schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(
schemaElementMatch.getElement().getType()))
.map(SchemaElementMatch::getElement)
.map(SchemaElement::getId)
.filter(schemaElementMatch -> SchemaElementType.VALUE
.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.map(SchemaElementMatch::getElement).map(SchemaElement::getId)
.collect(Collectors.toSet());
}
@@ -270,10 +255,8 @@ public class PluginManager {
if (CollectionUtils.isEmpty(paramOptions)) {
return Lists.newArrayList();
}
return paramOptions.stream()
.filter(
paramOption ->
ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType()))
return paramOptions.stream().filter(
paramOption -> ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType()))
.collect(Collectors.toList());
}

View File

@@ -26,13 +26,10 @@ public class ParamOption {
* forward
*/
public enum ParamType {
CUSTOM,
SEMANTIC,
FORWARD
CUSTOM, SEMANTIC, FORWARD
}
public enum OptionType {
REQUIRED,
OPTIONAL
REQUIRED, OPTIONAL
}
}

View File

@@ -43,32 +43,23 @@ public abstract class PluginSemanticQuery {
protected Map<String, Object> getElementMap(PluginParseResult pluginParseResult) {
Map<String, Object> elementValueMap = new HashMap<>();
Map<Long, Object> filterValueMap = getFilterMap(pluginParseResult);
List<SchemaElementMatch> schemaElementMatchList =
parseInfo.getElementMatches().stream()
List<SchemaElementMatch> schemaElementMatchList = parseInfo.getElementMatches().stream()
.filter(schemaElementMatch -> schemaElementMatch.getFrequency() != null)
.sorted(
Comparator.comparingLong(SchemaElementMatch::getFrequency)
.reversed())
.sorted(Comparator.comparingLong(SchemaElementMatch::getFrequency).reversed())
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(schemaElementMatchList)) {
schemaElementMatchList.stream()
.filter(
schemaElementMatch ->
SchemaElementType.VALUE.equals(
schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(
schemaElementMatch.getElement().getType()))
schemaElementMatchList.stream().filter(schemaElementMatch -> SchemaElementType.VALUE
.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.filter(schemaElementMatch -> schemaElementMatch.getSimilarity() == 1.0)
.forEach(
schemaElementMatch -> {
.forEach(schemaElementMatch -> {
Object queryFilterValue =
filterValueMap.get(schemaElementMatch.getElement().getId());
if (queryFilterValue != null) {
if (String.valueOf(queryFilterValue)
.equals(String.valueOf(schemaElementMatch.getWord()))) {
elementValueMap.put(
String.valueOf(
schemaElementMatch.getElement().getId()),
String.valueOf(schemaElementMatch.getElement().getId()),
schemaElementMatch.getWord());
}
} else {

View File

@@ -41,10 +41,8 @@ public class WebPageQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties();
PluginParseResult pluginParseResult =
JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)),
PluginParseResult.class);
PluginParseResult pluginParseResult = JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class);
WebPageResp webPageResponse = buildResponse(pluginParseResult);
queryResult.setResponse(webPageResponse);
queryResult.setQueryState(QueryState.SUCCESS);

View File

@@ -45,10 +45,8 @@ public class WebServiceQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties();
PluginParseResult pluginParseResult =
JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)),
PluginParseResult.class);
PluginParseResult pluginParseResult = JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class);
WebServiceResp webServiceResponse = buildResponse(pluginParseResult);
Object object = webServiceResponse.getResult();
// in order to show webServiceQuery result int frontend conveniently,
@@ -74,9 +72,8 @@ public class WebServiceQuery extends PluginSemanticQuery {
protected WebServiceResp buildResponse(PluginParseResult pluginParseResult) {
WebServiceResp webServiceResponse = new WebServiceResp();
ChatPlugin plugin = pluginParseResult.getPlugin();
WebBase webBase =
fillWebBaseResult(
JsonUtil.toObject(plugin.getConfig(), WebBase.class), pluginParseResult);
WebBase webBase = fillWebBaseResult(JsonUtil.toObject(plugin.getConfig(), WebBase.class),
pluginParseResult);
webServiceResponse.setWebBase(webBase);
List<ParamOption> paramOptions = webBase.getParamOptions();
Map<String, Object> params = new HashMap<>();

View File

@@ -41,17 +41,16 @@ public abstract class PluginRecognizer {
public abstract PluginRecallResult recallPlugin(ParseContext parseContext);
public void buildQuery(
ParseContext parseContext, ParseResp parseResp, PluginRecallResult pluginRecallResult) {
public void buildQuery(ParseContext parseContext, ParseResp parseResp,
PluginRecallResult pluginRecallResult) {
ChatPlugin plugin = pluginRecallResult.getPlugin();
Set<Long> dataSetIds = pluginRecallResult.getDataSetIds();
if (plugin.isContainsAllDataSet()) {
dataSetIds = Sets.newHashSet(-1L);
}
for (Long dataSetId : dataSetIds) {
SemanticParseInfo semanticParseInfo =
buildSemanticParseInfo(
dataSetId, plugin, parseContext, pluginRecallResult.getDistance());
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(dataSetId, plugin,
parseContext, pluginRecallResult.getDistance());
semanticParseInfo.setQueryMode(plugin.getType());
semanticParseInfo.setScore(pluginRecallResult.getScore());
parseResp.getSelectedParses().add(semanticParseInfo);
@@ -62,8 +61,8 @@ public abstract class PluginRecognizer {
return PluginManager.getPluginAgentCanSupport(parseContext);
}
protected SemanticParseInfo buildSemanticParseInfo(
Long dataSetId, ChatPlugin plugin, ParseContext parseContext, double distance) {
protected SemanticParseInfo buildSemanticParseInfo(Long dataSetId, ChatPlugin plugin,
ParseContext parseContext, double distance) {
List<SchemaElementMatch> schemaElementMatches =
parseContext.getMapInfo().getMatchedElements(dataSetId);
QueryFilters queryFilters = parseContext.getQueryFilters();
@@ -97,14 +96,10 @@ public abstract class PluginRecognizer {
return;
}
schemaElementMatches.stream()
.filter(
schemaElementMatch ->
SchemaElementType.VALUE.equals(
schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(
schemaElementMatch.getElement().getType()))
.forEach(
schemaElementMatch -> {
.filter(schemaElementMatch -> SchemaElementType.VALUE
.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.forEach(schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());

View File

@@ -53,12 +53,8 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer {
plugin.setParseMode(ParseMode.EMBEDDING_RECALL);
double similarity = embeddingRetrieval.getSimilarity();
double score = parseContext.getQueryText().length() * similarity;
return PluginRecallResult.builder()
.plugin(plugin)
.dataSetIds(dataSetList)
.score(score)
.distance(similarity)
.build();
return PluginRecallResult.builder().plugin(plugin).dataSetIds(dataSetList)
.score(score).distance(similarity).build();
}
}
return null;
@@ -71,11 +67,8 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer {
List<Retrieval> embeddingRetrievals = embeddingResp.getRetrieval();
if (!CollectionUtils.isEmpty(embeddingRetrievals)) {
embeddingRetrievals =
embeddingRetrievals.stream()
.sorted(
Comparator.comparingDouble(
o -> Math.abs(o.getSimilarity())))
embeddingRetrievals = embeddingRetrievals.stream()
.sorted(Comparator.comparingDouble(o -> Math.abs(o.getSimilarity())))
.collect(Collectors.toList());
embeddingResp.setRetrieval(embeddingRetrievals);
}

View File

@@ -1,4 +1,5 @@
package com.tencent.supersonic.chat.server.processor;
/** A ResultProcessor wraps things up before returning results to users. */
public interface ResultProcessor {}
public interface ResultProcessor {
}

View File

@@ -52,28 +52,20 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
List<Long> drillDownDimensions = Lists.newArrayList();
Set<SchemaElement> metricElements = dataSetSchema.getMetrics();
if (!CollectionUtils.isEmpty(metricElements)) {
Optional<SchemaElement> metric =
metricElements.stream()
.filter(
schemaElement ->
metricId.equals(schemaElement.getId())
&& !CollectionUtils.isEmpty(
schemaElement
.getRelatedSchemaElements()))
Optional<SchemaElement> metric = metricElements.stream()
.filter(schemaElement -> metricId.equals(schemaElement.getId())
&& !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements()))
.findFirst();
if (metric.isPresent()) {
drillDownDimensions =
metric.get().getRelatedSchemaElements().stream()
.map(RelatedSchemaElement::getDimensionId)
.collect(Collectors.toList());
drillDownDimensions = metric.get().getRelatedSchemaElements().stream()
.map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList());
}
}
final List<Long> drillDownDimensionsFinal = drillDownDimensions;
return dataSetSchema.getDimensions().stream()
.filter(dim -> filterDimension(drillDownDimensionsFinal, dim))
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(recommend_dimension_size)
.collect(Collectors.toList());
.limit(recommend_dimension_size).collect(Collectors.toList());
}
private boolean filterDimension(List<Long> drillDownDimensions, SchemaElement dimension) {

View File

@@ -69,19 +69,14 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
queryResult.setAggregateInfo(aggregateInfo);
}
public AggregateInfo getAggregateInfo(
User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) {
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo,
QueryResult queryResult) {
Set<String> resultMetricNames = new HashSet<>();
queryResult.getQueryColumns().stream()
.forEach(
c ->
resultMetricNames.addAll(
SqlSelectHelper.getColumnFromExpr(c.getNameEn())));
Optional<SchemaElement> ratioMetric =
semanticParseInfo.getMetrics().stream()
.filter(m -> resultMetricNames.contains(m.getBizName()))
.findFirst();
queryResult.getQueryColumns().stream().forEach(
c -> resultMetricNames.addAll(SqlSelectHelper.getColumnFromExpr(c.getNameEn())));
Optional<SchemaElement> ratioMetric = semanticParseInfo.getMetrics().stream()
.filter(m -> resultMetricNames.contains(m.getBizName())).findFirst();
AggregateInfo aggregateInfo = new AggregateInfo();
if (!ratioMetric.isPresent()) {
@@ -90,20 +85,15 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
try {
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp =
queryResult.getQueryResults().stream()
.filter(r -> r.containsKey(dateField))
.map(r -> r.get(dateField).toString())
.sorted(Comparator.reverseOrder())
.findFirst();
Optional<String> lastDayOp = queryResult.getQueryResults().stream()
.filter(r -> r.containsKey(dateField)).map(r -> r.get(dateField).toString())
.sorted(Comparator.reverseOrder()).findFirst();
if (!lastDayOp.isPresent()) {
return new AggregateInfo();
}
Optional<Map<String, Object>> lastValue =
queryResult.getQueryResults().stream()
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get()))
.findFirst();
Optional<Map<String, Object>> lastValue = queryResult.getQueryResults().stream()
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst();
MetricInfo metricInfo = new MetricInfo();
metricInfo.setStatistics(new HashMap<>());
@@ -115,23 +105,11 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
metricInfo.setDate(lastValue.get().get(dateField).toString());
CompletableFuture<MetricInfo> metricInfoRoll =
CompletableFuture.supplyAsync(
() ->
queryRatio(
user,
semanticParseInfo,
ratioMetric.get(),
AggOperatorEnum.RATIO_ROLL,
queryResult));
CompletableFuture.supplyAsync(() -> queryRatio(user, semanticParseInfo,
ratioMetric.get(), AggOperatorEnum.RATIO_ROLL, queryResult));
CompletableFuture<MetricInfo> metricInfoOver =
CompletableFuture.supplyAsync(
() ->
queryRatio(
user,
semanticParseInfo,
ratioMetric.get(),
AggOperatorEnum.RATIO_OVER,
queryResult));
CompletableFuture.supplyAsync(() -> queryRatio(user, semanticParseInfo,
ratioMetric.get(), AggOperatorEnum.RATIO_OVER, queryResult));
CompletableFuture.allOf(metricInfoRoll, metricInfoOver);
metricInfo.setName(metricInfoRoll.get().getName());
metricInfo.setValue(metricInfoRoll.get().getValue());
@@ -145,19 +123,15 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
}
@SneakyThrows
private MetricInfo queryRatio(
User user,
SemanticParseInfo semanticParseInfo,
SchemaElement metric,
AggOperatorEnum aggOperatorEnum,
QueryResult queryResult) {
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo,
SchemaElement metric, AggOperatorEnum aggOperatorEnum, QueryResult queryResult) {
QueryStructReq queryStructReq =
QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum);
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField)));
queryStructReq.setDateInfo(
getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult));
queryStructReq
.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult));
queryStructReq.setConvertToSql(false);
SemanticLayerService queryService = ContextUtils.getBean(SemanticLayerService.class);
SemanticQueryResp queryResp = queryService.queryByReq(queryStructReq, user);
@@ -168,25 +142,21 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
}
Map<String, Object> result = queryResp.getResultList().get(0);
Optional<QueryColumn> valueColumn =
queryResp.getColumns().stream()
.filter(c -> c.getNameEn().equals(metric.getBizName()))
.findFirst();
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst();
if (!valueColumn.isPresent()) {
return metricInfo;
}
String valueField =
String.format(
"%s_%s", valueColumn.get().getNameEn(), aggOperatorEnum.getOperator());
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(),
aggOperatorEnum.getOperator());
if (result.containsKey(valueColumn.get().getNameEn())) {
DecimalFormat df = new DecimalFormat("#.####");
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn())));
}
String ratio = "";
if (Objects.nonNull(result.get(valueField))) {
ratio =
String.format("%.2f", (Double.valueOf(result.get(valueField).toString()) * 100))
ratio = String.format("%.2f", (Double.valueOf(result.get(valueField).toString()) * 100))
+ "%";
}
String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName();
@@ -199,28 +169,20 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
statisticsRollName = RatioOverType.WEEK_ON_WEEK.getShowName();
statisticsOverName = RatioOverType.MONTH_ON_WEEK.getShowName();
}
metricInfo
.getStatistics()
.put(
aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
? statisticsRollName
: statisticsOverName,
ratio);
metricInfo.getStatistics()
.put(aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? statisticsRollName
: statisticsOverName, ratio);
metricInfo.setName(metric.getName());
return metricInfo;
}
private DateConf getRatioDateConf(
AggOperatorEnum aggOperatorEnum,
SemanticParseInfo semanticParseInfo,
QueryResult queryResult) {
private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum,
SemanticParseInfo semanticParseInfo, QueryResult queryResult) {
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp =
queryResult.getQueryResults().stream()
.map(r -> r.get(dateField).toString())
.sorted(Comparator.reverseOrder())
.findFirst();
queryResult.getQueryResults().stream().map(r -> r.get(dateField).toString())
.sorted(Comparator.reverseOrder()).findFirst();
if (!lastDayOp.isPresent()) {
return semanticParseInfo.getDateInfo();
@@ -236,29 +198,23 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
DateTimeFormatter formatter =
DateUtils.getDateFormatter(lastDay, new String[] {DAY_FORMAT, DAY_FORMAT_INT});
LocalDate end = LocalDate.parse(lastDay, formatter);
start =
aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
? end.minusDays(1).format(formatter)
: end.minusWeeks(1).format(formatter);
}
if (DatePeriodEnum.WEEK.equals(semanticParseInfo.getDateInfo().getPeriod())) {
DateTimeFormatter formatter =
DateUtils.getTimeFormatter(
lastDay,
DateTimeFormatter formatter = DateUtils.getTimeFormatter(lastDay,
new String[] {TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT});
LocalDateTime end = LocalDateTime.parse(lastDay, formatter);
start =
aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
? end.minusWeeks(1).format(formatter)
: end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter);
}
if (DatePeriodEnum.MONTH.equals(semanticParseInfo.getDateInfo().getPeriod())) {
DateTimeFormatter formatter =
DateUtils.getDateFormatter(
lastDay, new String[] {MONTH_FORMAT, MONTH_FORMAT_INT});
DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay,
new String[] {MONTH_FORMAT, MONTH_FORMAT_INT});
YearMonth end = YearMonth.parse(lastDay, formatter);
start =
aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
? end.minusMonths(1).format(formatter)
: end.minusYears(1).format(formatter);
}

View File

@@ -45,32 +45,23 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor {
List<String> metricNames =
Collections.singletonList(parseInfo.getMetrics().iterator().next().getName());
Map<String, Object> filterCondition = new HashMap<>();
filterCondition.put(
"modelId", parseInfo.getMetrics().iterator().next().getDataSetId().toString());
filterCondition.put("modelId",
parseInfo.getMetrics().iterator().next().getDataSetId().toString());
filterCondition.put("type", SchemaElementType.METRIC.name());
RetrieveQuery retrieveQuery =
RetrieveQuery.builder()
.queryTextsList(metricNames)
.filterCondition(filterCondition)
.queryEmbeddings(null)
.build();
RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames)
.filterCondition(filterCondition).queryEmbeddings(null).build();
MetaEmbeddingService metaEmbeddingService =
ContextUtils.getBean(MetaEmbeddingService.class);
List<RetrieveQueryResult> retrieveQueryResults =
metaEmbeddingService.retrieveQuery(
List<RetrieveQueryResult> retrieveQueryResults = metaEmbeddingService.retrieveQuery(
retrieveQuery, METRIC_RECOMMEND_SIZE + 1, new HashMap<>(), new HashSet<>());
if (CollectionUtils.isEmpty(retrieveQueryResults)) {
return;
}
List<Retrieval> retrievals =
retrieveQueryResults.stream()
List<Retrieval> retrievals = retrieveQueryResults.stream()
.flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream())
.sorted(Comparator.comparingDouble(Retrieval::getSimilarity))
.distinct()
.sorted(Comparator.comparingDouble(Retrieval::getSimilarity)).distinct()
.collect(Collectors.toList());
Set<Long> metricIds =
parseInfo.getMetrics().stream()
.map(SchemaElement::getId)
Set<Long> metricIds = parseInfo.getMetrics().stream().map(SchemaElement::getId)
.collect(Collectors.toSet());
int metricOrder = 0;
for (SchemaElement metric : parseInfo.getMetrics()) {
@@ -79,22 +70,14 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor {
for (Retrieval retrieval : retrievals) {
if (!metricIds.contains(Retrieval.getLongId(retrieval.getId()))) {
if (Objects.nonNull(retrieval.getMetadata().get("id"))) {
String idStr =
retrieval
.getMetadata()
.get("id")
.toString()
String idStr = retrieval.getMetadata().get("id").toString()
.replaceAll(DictWordType.NATURE_SPILT, "");
retrieval.getMetadata().put("id", idStr);
}
String metaStr = JSONObject.toJSONString(retrieval.getMetadata());
SchemaElement schemaElement = JSONObject.parseObject(metaStr, SchemaElement.class);
if (retrieval.getMetadata().containsKey("dataSetId")) {
String dataSetId =
retrieval
.getMetadata()
.get("dataSetId")
.toString()
String dataSetId = retrieval.getMetadata().get("dataSetId").toString()
.replace(Constants.UNDERLINE, "");
schemaElement.setDataSetId(Long.parseLong(dataSetId));
}

View File

@@ -43,13 +43,8 @@ public class QueryRecommendProcessor implements ParseResultProcessor {
String memoryCollectionName = embeddingConfig.getMemoryCollectionName(agentId);
List<Text2SQLExemplar> exemplars =
exemplarService.recallExemplars(memoryCollectionName, queryText, 5);
return exemplars.stream()
.map(
sqlExemplar ->
SimilarQueryRecallResp.builder()
.queryText(sqlExemplar.getQuestion())
.build())
.collect(Collectors.toList());
return exemplars.stream().map(sqlExemplar -> SimilarQueryRecallResp.builder()
.queryText(sqlExemplar.getQuestion()).build()).collect(Collectors.toList());
}
private ChatQueryDO getChatQuery(Long queryId) {

View File

@@ -11,11 +11,7 @@ public class TimeCostProcessor implements ParseResultProcessor {
@Override
public void process(ParseContext parseContext, ParseResp parseResp) {
long parseStartTime = parseResp.getParseTimeCost().getParseStartTime();
parseResp
.getParseTimeCost()
.setParseTime(
System.currentTimeMillis()
- parseStartTime
parseResp.getParseTimeCost().setParseTime(System.currentTimeMillis() - parseStartTime
- parseResp.getParseTimeCost().getSqlTime());
}
}

View File

@@ -26,21 +26,18 @@ import java.util.Map;
@RequestMapping({"/api/chat/agent", "/openapi/chat/agent"})
public class AgentController {
@Autowired private AgentService agentService;
@Autowired
private AgentService agentService;
@PostMapping
public Agent createAgent(
@RequestBody Agent agent,
HttpServletRequest httpServletRequest,
public Agent createAgent(@RequestBody Agent agent, HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
return agentService.createAgent(agent, user);
}
@PutMapping
public Agent updateAgent(
@RequestBody Agent agent,
HttpServletRequest httpServletRequest,
public Agent updateAgent(@RequestBody Agent agent, HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
return agentService.updateAgent(agent, user);

View File

@@ -29,33 +29,29 @@ import java.util.List;
@RequestMapping({"/api/chat/conf", "/openapi/chat/conf"})
public class ChatConfigController {
@Autowired private ConfigService configService;
@Autowired
private ConfigService configService;
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@PostMapping
public Long addChatConfig(
@RequestBody ChatConfigBaseReq extendBaseCmd,
HttpServletRequest request,
HttpServletResponse response) {
public Long addChatConfig(@RequestBody ChatConfigBaseReq extendBaseCmd,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return configService.addConfig(extendBaseCmd, user);
}
@PutMapping
public Long editModelExtend(
@RequestBody ChatConfigEditReqReq extendEditCmd,
HttpServletRequest request,
HttpServletResponse response) {
public Long editModelExtend(@RequestBody ChatConfigEditReqReq extendEditCmd,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return configService.editConfig(extendEditCmd, user);
}
@PostMapping("/search")
public List<ChatConfigResp> search(
@RequestBody ChatConfigFilter filter,
HttpServletRequest request,
HttpServletResponse response) {
public List<ChatConfigResp> search(@RequestBody ChatConfigFilter filter,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return configService.search(filter, user);
}

View File

@@ -25,14 +25,13 @@ import java.util.List;
@RequestMapping({"/api/chat/manage", "/openapi/chat/manage"})
public class ChatController {
@Autowired private ChatManageService chatService;
@Autowired
private ChatManageService chatService;
@PostMapping("/save")
public Boolean save(
@RequestParam(value = "chatName") String chatName,
public Boolean save(@RequestParam(value = "chatName") String chatName,
@RequestParam(value = "agentId", required = false) Integer agentId,
HttpServletRequest request,
HttpServletResponse response) {
HttpServletRequest request, HttpServletResponse response) {
chatService.addChat(UserHolder.findUser(request, response), chatName, agentId);
return true;
}
@@ -40,50 +39,42 @@ public class ChatController {
@GetMapping("/getAll")
public List<ChatDO> getAllConversions(
@RequestParam(value = "agentId", required = false) Integer agentId,
HttpServletRequest request,
HttpServletResponse response) {
HttpServletRequest request, HttpServletResponse response) {
String userName = UserHolder.findUser(request, response).getName();
return chatService.getAll(userName, agentId);
}
@PostMapping("/delete")
public Boolean deleteConversion(
@RequestParam(value = "chatId") long chatId,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean deleteConversion(@RequestParam(value = "chatId") long chatId,
HttpServletRequest request, HttpServletResponse response) {
String userName = UserHolder.findUser(request, response).getName();
return chatService.deleteChat(chatId, userName);
}
@PostMapping("/updateChatName")
public Boolean updateConversionName(
@RequestParam(value = "chatId") Long chatId,
@RequestParam(value = "chatName") String chatName,
HttpServletRequest request,
public Boolean updateConversionName(@RequestParam(value = "chatId") Long chatId,
@RequestParam(value = "chatName") String chatName, HttpServletRequest request,
HttpServletResponse response) {
String userName = UserHolder.findUser(request, response).getName();
return chatService.updateChatName(chatId, chatName, userName);
}
@PostMapping("/updateQAFeedback")
public Boolean updateQAFeedback(
@RequestParam(value = "id") Integer id,
public Boolean updateQAFeedback(@RequestParam(value = "id") Integer id,
@RequestParam(value = "score") Integer score,
@RequestParam(value = "feedback", required = false) String feedback) {
return chatService.updateFeedback(id, score, feedback);
}
@PostMapping("/updateChatIsTop")
public Boolean updateConversionIsTop(
@RequestParam(value = "chatId") Long chatId, @RequestParam(value = "isTop") int isTop) {
public Boolean updateConversionIsTop(@RequestParam(value = "chatId") Long chatId,
@RequestParam(value = "isTop") int isTop) {
return chatService.updateChatIsTop(chatId, isTop);
}
@PostMapping("/pageQueryInfo")
public PageInfo<QueryResp> pageQueryInfo(
@RequestBody PageQueryInfoReq pageQueryInfoCommand,
@RequestParam(value = "chatId") long chatId,
HttpServletRequest request,
public PageInfo<QueryResp> pageQueryInfo(@RequestBody PageQueryInfoReq pageQueryInfoCommand,
@RequestParam(value = "chatId") long chatId, HttpServletRequest request,
HttpServletResponse response) {
pageQueryInfoCommand.setUserName(UserHolder.findUser(request, response).getName());
return chatService.queryInfo(pageQueryInfoCommand, chatId);
@@ -95,8 +86,7 @@ public class ChatController {
}
@PostMapping("/queryShowCase")
public ShowCaseResp queryShowCase(
@RequestBody PageQueryInfoReq pageQueryInfoCommand,
public ShowCaseResp queryShowCase(@RequestBody PageQueryInfoReq pageQueryInfoCommand,
@RequestParam(value = "agentId") int agentId) {
return chatService.queryShowCase(pageQueryInfoCommand, agentId);
}

View File

@@ -27,43 +27,33 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping({"/api/chat/query", "/openapi/chat/query"})
public class ChatQueryController {
@Autowired private ChatQueryService chatQueryService;
@Autowired
private ChatQueryService chatQueryService;
@PostMapping("search")
public Object search(
@RequestBody ChatParseReq chatParseReq,
HttpServletRequest request,
public Object search(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
HttpServletResponse response) {
chatParseReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.search(chatParseReq);
}
@PostMapping("parse")
public Object parse(
@RequestBody ChatParseReq chatParseReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object parse(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
chatParseReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performParsing(chatParseReq);
}
@PostMapping("execute")
public Object execute(
@RequestBody ChatExecuteReq chatExecuteReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object execute(@RequestBody ChatExecuteReq chatExecuteReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
chatExecuteReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performExecution(chatExecuteReq);
}
@PostMapping("/")
public Object query(
@RequestBody ChatParseReq chatParseReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object query(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
chatParseReq.setUser(user);
ParseResp parseResp = chatQueryService.performParsing(chatParseReq);
@@ -80,22 +70,16 @@ public class ChatQueryController {
}
@PostMapping("queryData")
public Object queryData(
@RequestBody ChatQueryDataReq chatQueryDataReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryData(@RequestBody ChatQueryDataReq chatQueryDataReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
chatQueryDataReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.queryData(chatQueryDataReq, UserHolder.findUser(request, response));
}
@PostMapping("queryDimensionValue")
public Object queryDimensionValue(
@RequestBody @Valid DimensionValueReq dimensionValueReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
return chatQueryService.queryDimensionValue(
dimensionValueReq, UserHolder.findUser(request, response));
public Object queryDimensionValue(@RequestBody @Valid DimensionValueReq dimensionValueReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
return chatQueryService.queryDimensionValue(dimensionValueReq,
UserHolder.findUser(request, response));
}
}

View File

@@ -21,13 +21,12 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping({"/api/chat/memory"})
public class MemoryController {
@Autowired private MemoryService memoryService;
@Autowired
private MemoryService memoryService;
@PostMapping("/updateMemory")
public Boolean updateMemory(
@RequestBody ChatMemoryUpdateReq chatMemoryUpdateReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean updateMemory(@RequestBody ChatMemoryUpdateReq chatMemoryUpdateReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
memoryService.updateMemory(chatMemoryUpdateReq, user);
return true;

View File

@@ -25,23 +25,20 @@ import java.util.List;
@RequestMapping("/api/chat/plugin")
public class PluginController {
@Autowired protected PluginService pluginService;
@Autowired
protected PluginService pluginService;
@PostMapping
public boolean createPlugin(
@RequestBody ChatPlugin plugin,
HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
public boolean createPlugin(@RequestBody ChatPlugin plugin,
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
pluginService.createPlugin(plugin, user);
return true;
}
@PutMapping
public boolean updatePlugin(
@RequestBody ChatPlugin plugin,
HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
public boolean updatePlugin(@RequestBody ChatPlugin plugin,
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
pluginService.updatePlugin(plugin, user);
return true;
@@ -59,18 +56,16 @@ public class PluginController {
}
@PostMapping("/query")
List<ChatPlugin> query(
@RequestBody PluginQueryReq pluginQueryReq,
HttpServletRequest httpServletRequest,
HttpServletResponse httpServletResponse) {
List<ChatPlugin> query(@RequestBody PluginQueryReq pluginQueryReq,
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
return pluginService.queryWithAuthCheck(pluginQueryReq, user);
}
@AuthenticationIgnore
@PostMapping("/pluginDemo")
public String pluginDemo(
@RequestParam("queryText") String queryText, @RequestBody Object object) {
public String pluginDemo(@RequestParam("queryText") String queryText,
@RequestBody Object object) {
return String.format("已收到您的问题:%s, 但这只是一个demo~", queryText);
}
}

View File

@@ -33,9 +33,11 @@ import java.util.stream.Collectors;
@Service
public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implements AgentService {
@Autowired private MemoryService memoryService;
@Autowired
private MemoryService memoryService;
@Autowired private ChatQueryService chatQueryService;
@Autowired
private ChatQueryService chatQueryService;
private ExecutorService executorService = Executors.newFixedThreadPool(1);
@@ -98,8 +100,7 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
}
private synchronized void doExecuteAgentExamples(Agent agent) {
if (!agent.containsLLMTool()
|| !LLMConnHelper.testConnection(agent.getModelConfig())
if (!agent.containsLLMTool() || !LLMConnHelper.testConnection(agent.getModelConfig())
|| CollectionUtils.isEmpty(agent.getExamples())) {
return;
}
@@ -107,10 +108,8 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
List<String> examples = agent.getExamples();
ChatMemoryFilter chatMemoryFilter =
ChatMemoryFilter.builder().agentId(agent.getId()).questions(examples).build();
List<String> memoriesExisted =
memoryService.getMemories(chatMemoryFilter).stream()
.map(ChatMemoryDO::getQuestion)
.collect(Collectors.toList());
List<String> memoriesExisted = memoryService.getMemories(chatMemoryFilter).stream()
.map(ChatMemoryDO::getQuestion).collect(Collectors.toList());
for (String example : examples) {
if (memoriesExisted.contains(example)) {
continue;

View File

@@ -37,8 +37,10 @@ import java.util.stream.Collectors;
@Service
public class ChatManageServiceImpl implements ChatManageService {
@Autowired private ChatRepository chatRepository;
@Autowired private ChatQueryRepository chatQueryRepository;
@Autowired
private ChatRepository chatRepository;
@Autowired
private ChatQueryRepository chatQueryRepository;
@Override
public Long addChat(User user, String chatName, Integer agentId) {
@@ -121,8 +123,7 @@ public class ChatManageServiceImpl implements ChatManageService {
if (CollectionUtils.isEmpty(queryResps)) {
return showCaseResp;
}
queryResps.removeIf(
queryResp -> {
queryResps.removeIf(queryResp -> {
if (queryResp.getQueryResult() == null) {
return true;
}
@@ -135,15 +136,9 @@ public class ChatManageServiceImpl implements ChatManageService {
Map<String, Object> data = queryResp.getQueryResult().getQueryResults().get(0);
return CollectionUtils.isEmpty(data);
});
queryResps =
new ArrayList<>(
queryResps.stream()
.collect(
Collectors.toMap(
QueryResp::getQueryText,
Function.identity(),
(existing, replacement) -> existing,
LinkedHashMap::new))
queryResps = new ArrayList<>(queryResps.stream()
.collect(Collectors.toMap(QueryResp::getQueryText, Function.identity(),
(existing, replacement) -> existing, LinkedHashMap::new))
.values());
fillParseInfo(queryResps);
Map<Long, List<QueryResp>> showCaseMap =
@@ -166,16 +161,10 @@ public class ChatManageServiceImpl implements ChatManageService {
if (CollectionUtils.isEmpty(chatParseDOList)) {
continue;
}
List<SemanticParseInfo> parseInfos =
chatParseDOList.stream()
.map(
chatParseDO ->
JsonUtil.toObject(
chatParseDO.getParseInfo(),
List<SemanticParseInfo> parseInfos = chatParseDOList.stream()
.map(chatParseDO -> JsonUtil.toObject(chatParseDO.getParseInfo(),
SemanticParseInfo.class))
.sorted(
Comparator.comparingDouble(SemanticParseInfo::getScore)
.reversed())
.sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
.collect(Collectors.toList());
queryResp.setParseInfos(parseInfos);
}
@@ -188,10 +177,8 @@ public class ChatManageServiceImpl implements ChatManageService {
chatQueryDO.setQueryResult(JsonUtil.toString(queryResult));
chatQueryDO.setQueryState(1);
updateQuery(chatQueryDO);
chatRepository.updateLastQuestion(
chatExecuteReq.getChatId().longValue(),
chatExecuteReq.getQueryText(),
getCurrentTime());
chatRepository.updateLastQuestion(chatExecuteReq.getChatId().longValue(),
chatExecuteReq.getQueryText(), getCurrentTime());
return chatQueryDO;
}

View File

@@ -78,10 +78,14 @@ import java.util.stream.Collectors;
@Service
public class ChatQueryServiceImpl implements ChatQueryService {
@Autowired private ChatManageService chatManageService;
@Autowired private ChatLayerService chatLayerService;
@Autowired private SemanticLayerService semanticLayerService;
@Autowired private AgentService agentService;
@Autowired
private ChatManageService chatManageService;
@Autowired
private ChatLayerService chatLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired
private AgentService agentService;
private List<ChatQueryParser> chatQueryParsers = ComponentFactory.getChatParsers();
private List<ChatQueryExecutor> chatQueryExecutors = ComponentFactory.getChatExecutors();
@@ -149,11 +153,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
chatParseReq.setUser(User.getFakeUser());
ParseResp parseResp = performParsing(chatParseReq);
if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) {
log.debug(
"chatId:{}, agentId:{}, queryText:{}, parseResp.getSelectedParses() is empty",
chatId,
agentId,
queryText);
log.debug("chatId:{}, agentId:{}, queryText:{}, parseResp.getSelectedParses() is empty",
chatId, agentId, queryText);
return null;
}
ChatExecuteReq executeReq = new ChatExecuteReq();
@@ -184,9 +185,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
private ExecuteContext buildExecuteContext(ChatExecuteReq chatExecuteReq) {
ExecuteContext executeContext = new ExecuteContext();
BeanMapper.mapper(chatExecuteReq, executeContext);
SemanticParseInfo parseInfo =
chatManageService.getParseInfo(
chatExecuteReq.getQueryId(), chatExecuteReq.getParseId());
SemanticParseInfo parseInfo = chatManageService.getParseInfo(chatExecuteReq.getQueryId(),
chatExecuteReq.getParseId());
Agent agent = agentService.getAgent(chatExecuteReq.getAgentId());
executeContext.setAgent(agent);
executeContext.setParseInfo(parseInfo);
@@ -222,12 +222,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return SqlSelectHelper.getAllSelectFields(sqlInfo.getCorrectedS2SQL());
}
private void handleLLMQueryMode(
ChatQueryDataReq chatQueryDataReq,
SemanticQuery semanticQuery,
DataSetSchema dataSetSchema,
User user)
throws Exception {
private void handleLLMQueryMode(ChatQueryDataReq chatQueryDataReq, SemanticQuery semanticQuery,
DataSetSchema dataSetSchema, User user) throws Exception {
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
List<String> fields = getFieldsFromSql(parseInfo);
if (checkMetricReplace(fields, chatQueryDataReq.getMetrics())) {
@@ -245,16 +241,16 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
}
private void handleRuleQueryMode(
SemanticQuery semanticQuery, DataSetSchema dataSetSchema, User user) {
private void handleRuleQueryMode(SemanticQuery semanticQuery, DataSetSchema dataSetSchema,
User user) {
log.info("rule begin replace metrics and revise filters!");
validFilter(semanticQuery.getParseInfo().getDimensionFilters());
validFilter(semanticQuery.getParseInfo().getMetricFilters());
semanticQuery.initS2Sql(dataSetSchema, user);
}
private QueryResult executeQuery(
SemanticQuery semanticQuery, User user, DataSetSchema dataSetSchema) throws Exception {
private QueryResult executeQuery(SemanticQuery semanticQuery, User user,
DataSetSchema dataSetSchema) throws Exception {
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
QueryResult queryResult = doExecution(semanticQueryReq, parseInfo.getQueryMode(), user);
@@ -275,8 +271,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return !oriFields.containsAll(metricNames);
}
private String reviseCorrectS2SQL(
ChatQueryDataReq queryData, SemanticParseInfo parseInfo, DataSetSchema dataSetSchema) {
private String reviseCorrectS2SQL(ChatQueryDataReq queryData, SemanticParseInfo parseInfo,
DataSetSchema dataSetSchema) {
String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL();
log.info("correctorSql before replacing:{}", correctorSql);
// get where filter and having filter
@@ -286,21 +282,12 @@ public class ChatQueryServiceImpl implements ChatQueryService {
// replace where filter
List<Expression> addWhereConditions = new ArrayList<>();
Set<String> removeWhereFieldNames =
updateFilters(
whereExpressionList,
queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(),
addWhereConditions);
updateFilters(whereExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addWhereConditions);
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
Set<String> removeDataFieldNames =
updateDateInfo(
queryData,
parseInfo,
dataSetSchema,
filedNameToValueMap,
whereExpressionList,
addWhereConditions);
Set<String> removeDataFieldNames = updateDateInfo(queryData, parseInfo, dataSetSchema,
filedNameToValueMap, whereExpressionList, addWhereConditions);
removeWhereFieldNames.addAll(removeDataFieldNames);
correctorSql = SqlReplaceHelper.replaceValue(correctorSql, filedNameToValueMap);
@@ -311,11 +298,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
SqlSelectHelper.getHavingExpressions(correctorSql);
List<Expression> addHavingConditions = new ArrayList<>();
Set<String> removeHavingFieldNames =
updateFilters(
havingExpressionList,
queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(),
addHavingConditions);
updateFilters(havingExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addHavingConditions);
correctorSql = SqlReplaceHelper.replaceHavingValue(correctorSql, new HashMap<>());
correctorSql = SqlRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames);
@@ -326,9 +310,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
private void replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) {
List<String> oriMetrics =
parseInfo.getMetrics().stream()
.map(SchemaElement::getName)
List<String> oriMetrics = parseInfo.getMetrics().stream().map(SchemaElement::getName)
.collect(Collectors.toList());
String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL();
log.info("before replaceMetrics:{}", correctorSql);
@@ -362,20 +344,15 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return queryResult;
}
private Set<String> updateDateInfo(
ChatQueryDataReq queryData,
SemanticParseInfo parseInfo,
DataSetSchema dataSetSchema,
Map<String, Map<String, String>> filedNameToValueMap,
List<FieldExpression> fieldExpressionList,
List<Expression> addConditions) {
private Set<String> updateDateInfo(ChatQueryDataReq queryData, SemanticParseInfo parseInfo,
DataSetSchema dataSetSchema, Map<String, Map<String, String>> filedNameToValueMap,
List<FieldExpression> fieldExpressionList, List<Expression> addConditions) {
Set<String> removeFieldNames = new HashSet<>();
if (Objects.isNull(queryData.getDateInfo())) {
return removeFieldNames;
}
if (queryData.getDateInfo().getUnit() > 1) {
queryData
.getDateInfo()
queryData.getDateInfo()
.setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1));
queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(0));
}
@@ -386,16 +363,10 @@ public class ChatQueryServiceImpl implements ChatQueryService {
// first remove,then add
removeFieldNames.add(partitionDimension.getName());
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addTimeFilters(
queryData.getDateInfo().getStartDate(),
greaterThanEquals,
addConditions,
partitionDimension);
addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals,
addConditions, partitionDimension);
MinorThanEquals minorThanEquals = new MinorThanEquals();
addTimeFilters(
queryData.getDateInfo().getEndDate(),
minorThanEquals,
addConditions,
addTimeFilters(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions,
partitionDimension);
break;
}
@@ -403,8 +374,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
for (FieldExpression fieldExpression : fieldExpressionList) {
for (QueryFilter queryFilter : queryData.getDimensionFilters()) {
if (queryFilter.getOperator().equals(FilterOperatorEnum.LIKE)
&& FilterOperatorEnum.LIKE
.getValue()
&& FilterOperatorEnum.LIKE.getValue()
.equalsIgnoreCase(fieldExpression.getOperator())) {
Map<String, String> replaceMap = new HashMap<>();
String preValue = fieldExpression.getFieldValue().toString();
@@ -425,11 +395,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return removeFieldNames;
}
private <T extends ComparisonOperator> void addTimeFilters(
String date,
T comparisonExpression,
List<Expression> addConditions,
SchemaElement partitionDimension) {
private <T extends ComparisonOperator> void addTimeFilters(String date, T comparisonExpression,
List<Expression> addConditions, SchemaElement partitionDimension) {
Column column = new Column(partitionDimension.getName());
StringValue stringValue = new StringValue(date);
comparisonExpression.setLeftExpression(column);
@@ -437,10 +404,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
addConditions.add(comparisonExpression);
}
private Set<String> updateFilters(
List<FieldExpression> fieldExpressionList,
Set<QueryFilter> metricFilters,
Set<QueryFilter> contextMetricFilters,
private Set<String> updateFilters(List<FieldExpression> fieldExpressionList,
Set<QueryFilter> metricFilters, Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
Set<String> removeFieldNames = new HashSet<>();
if (CollectionUtils.isEmpty(metricFilters)) {
@@ -460,15 +425,13 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return removeFieldNames;
}
private void handleFilter(
QueryFilter dslQueryFilter,
Set<QueryFilter> contextMetricFilters,
private void handleFilter(QueryFilter dslQueryFilter, Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
FilterOperatorEnum operator = dslQueryFilter.getOperator();
if (operator == FilterOperatorEnum.IN) {
addWhereInFilters(
dslQueryFilter, new InExpression(), contextMetricFilters, addConditions);
addWhereInFilters(dslQueryFilter, new InExpression(), contextMetricFilters,
addConditions);
} else {
ComparisonOperator expression = FilterOperatorEnum.createExpression(operator);
if (Objects.nonNull(expression)) {
@@ -478,11 +441,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
// add in condition to sql where condition
private void addWhereInFilters(
QueryFilter dslQueryFilter,
InExpression inExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
private void addWhereInFilters(QueryFilter dslQueryFilter, InExpression inExpression,
Set<QueryFilter> contextMetricFilters, List<Expression> addConditions) {
Column column = new Column(dslQueryFilter.getName());
ParenthesedExpressionList parenthesedExpressionList = new ParenthesedExpressionList<>();
List<String> valueList =
@@ -490,18 +450,14 @@ public class ChatQueryServiceImpl implements ChatQueryService {
if (CollectionUtils.isEmpty(valueList)) {
return;
}
valueList.stream()
.forEach(
o -> {
valueList.stream().forEach(o -> {
StringValue stringValue = new StringValue(o);
parenthesedExpressionList.add(stringValue);
});
inExpression.setLeftExpression(column);
inExpression.setRightExpression(parenthesedExpressionList);
addConditions.add(inExpression);
contextMetricFilters.stream()
.forEach(
o -> {
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
o.setOperator(dslQueryFilter.getOperator());
@@ -510,10 +466,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
// add where filter
private void addWhereFilters(
QueryFilter dslQueryFilter,
ComparisonOperator comparisonExpression,
Set<QueryFilter> contextMetricFilters,
private void addWhereFilters(QueryFilter dslQueryFilter,
ComparisonOperator comparisonExpression, Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
String columnName = dslQueryFilter.getName();
if (StringUtils.isNotBlank(dslQueryFilter.getFunction())) {
@@ -533,9 +487,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
comparisonExpression.setRightExpression(stringValue);
}
addConditions.add(comparisonExpression);
contextMetricFilters.stream()
.forEach(
o -> {
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
o.setOperator(dslQueryFilter.getOperator());
@@ -543,8 +495,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
});
}
private SemanticParseInfo mergeParseInfo(
SemanticParseInfo parseInfo, ChatQueryDataReq queryData) {
private SemanticParseInfo mergeParseInfo(SemanticParseInfo parseInfo,
ChatQueryDataReq queryData) {
if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return parseInfo;
}

View File

@@ -51,10 +51,8 @@ public class ConfigServiceImpl implements ConfigService {
private final ChatConfigHelper chatConfigHelper;
private final SemanticLayerService semanticLayerService;
public ConfigServiceImpl(
ChatConfigRepository chatConfigRepository,
ChatConfigHelper chatConfigHelper,
SemanticLayerService semanticLayerService) {
public ConfigServiceImpl(ChatConfigRepository chatConfigRepository,
ChatConfigHelper chatConfigHelper, SemanticLayerService semanticLayerService) {
this.chatConfigRepository = chatConfigRepository;
this.chatConfigHelper = chatConfigHelper;
this.semanticLayerService = semanticLayerService;
@@ -80,8 +78,7 @@ public class ConfigServiceImpl implements ConfigService {
@Override
public Long editConfig(ChatConfigEditReqReq configEditCmd, User user) {
log.info("[edit model extend] object:{}", JsonUtil.toString(configEditCmd, true));
if (Objects.isNull(configEditCmd)
|| Objects.isNull(configEditCmd.getId())
if (Objects.isNull(configEditCmd) || Objects.isNull(configEditCmd.getId())
&& Objects.isNull(configEditCmd.getModelId())) {
throw new RuntimeException(
"editConfig, id and modelId are not allowed to be empty at the same time");
@@ -107,13 +104,13 @@ public class ConfigServiceImpl implements ConfigService {
List<Long> blackDimIdList = new ArrayList<>();
if (Objects.nonNull(chatConfig.getChatAggConfig())
&& Objects.nonNull(chatConfig.getChatAggConfig().getVisibility())) {
blackDimIdList.addAll(
chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList());
blackDimIdList
.addAll(chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList());
}
if (Objects.nonNull(chatConfig.getChatDetailConfig())
&& Objects.nonNull(chatConfig.getChatDetailConfig().getVisibility())) {
blackDimIdList.addAll(
chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList());
blackDimIdList
.addAll(chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList());
}
List<Long> filterDimIdList =
blackDimIdList.stream().distinct().collect(Collectors.toList());
@@ -121,8 +118,8 @@ public class ConfigServiceImpl implements ConfigService {
List<Long> blackMetricIdList = new ArrayList<>();
if (Objects.nonNull(chatConfig.getChatAggConfig())
&& Objects.nonNull(chatConfig.getChatAggConfig().getVisibility())) {
blackMetricIdList.addAll(
chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList());
blackMetricIdList
.addAll(chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList());
}
if (Objects.nonNull(chatConfig.getChatDetailConfig())
&& Objects.nonNull(chatConfig.getChatDetailConfig().getVisibility())) {
@@ -138,20 +135,16 @@ public class ConfigServiceImpl implements ConfigService {
if (!CollectionUtils.isEmpty(blackDimIdList)) {
List<DimensionResp> dimensionRespList = semanticLayerService.getDimensions(metaFilter);
List<String> blackDimNameList =
dimensionRespList.stream()
.filter(o -> filterDimIdList.contains(o.getId()))
.map(SchemaItem::getName)
.collect(Collectors.toList());
dimensionRespList.stream().filter(o -> filterDimIdList.contains(o.getId()))
.map(SchemaItem::getName).collect(Collectors.toList());
itemNameVisibility.setBlackDimNameList(blackDimNameList);
}
if (!CollectionUtils.isEmpty(blackMetricIdList)) {
List<MetricResp> metricRespList = semanticLayerService.getMetrics(metaFilter);
List<String> blackMetricList =
metricRespList.stream()
.filter(o -> filterMetricIdList.contains(o.getId()))
.map(SchemaItem::getName)
.collect(Collectors.toList());
metricRespList.stream().filter(o -> filterMetricIdList.contains(o.getId()))
.map(SchemaItem::getName).collect(Collectors.toList());
itemNameVisibility.setBlackMetricNameList(blackMetricList);
}
return itemNameVisibility;
@@ -169,8 +162,8 @@ public class ConfigServiceImpl implements ConfigService {
return chatConfigRepository.getConfigByModelId(modelId);
}
private ItemVisibilityInfo fetchVisibilityDescByConfig(
ItemVisibility visibility, DataSetSchema modelSchema) {
private ItemVisibilityInfo fetchVisibilityDescByConfig(ItemVisibility visibility,
DataSetSchema modelSchema) {
ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo();
List<Long> dimIdAllList = chatConfigHelper.generateAllDimIdList(modelSchema);
@@ -186,15 +179,10 @@ public class ConfigServiceImpl implements ConfigService {
blackMetricIdList.addAll(visibility.getBlackMetricIdList());
}
}
List<Long> whiteMetricIdList =
metricIdAllList.stream()
.filter(
id ->
!blackMetricIdList.contains(id)
&& metricIdAllList.contains(id))
List<Long> whiteMetricIdList = metricIdAllList.stream()
.filter(id -> !blackMetricIdList.contains(id) && metricIdAllList.contains(id))
.collect(Collectors.toList());
List<Long> whiteDimIdList =
dimIdAllList.stream()
List<Long> whiteDimIdList = dimIdAllList.stream()
.filter(id -> !blackDimIdList.contains(id) && dimIdAllList.contains(id))
.collect(Collectors.toList());
@@ -232,10 +220,8 @@ public class ConfigServiceImpl implements ConfigService {
return chatConfigRich;
}
private ChatDetailRichConfigResp fillChatDetailRichConfig(
DataSetSchema modelSchema,
ChatConfigRichResp chatConfigRich,
ChatConfigResp chatConfigResp) {
private ChatDetailRichConfigResp fillChatDetailRichConfig(DataSetSchema modelSchema,
ChatConfigRichResp chatConfigRich, ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp)
|| Objects.isNull(chatConfigResp.getChatDetailConfig())) {
return null;
@@ -248,8 +234,7 @@ public class ConfigServiceImpl implements ConfigService {
detailRichConfig.setKnowledgeInfos(
fillKnowledgeBizName(chatDetailConfig.getKnowledgeInfos(), modelSchema));
detailRichConfig.setGlobalKnowledgeConfig(chatDetailConfig.getGlobalKnowledgeConfig());
detailRichConfig.setChatDefaultConfig(
fetchDefaultConfig(
detailRichConfig.setChatDefaultConfig(fetchDefaultConfig(
chatDetailConfig.getChatDefaultConfig(), modelSchema, itemVisibilityInfo));
return detailRichConfig;
@@ -261,18 +246,15 @@ public class ConfigServiceImpl implements ConfigService {
return entityRichInfo;
}
BeanUtils.copyProperties(entity, entityRichInfo);
Map<Long, SchemaElement> dimIdAndRespPair =
modelSchema.getDimensions().stream()
.collect(
Collectors.toMap(
SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId()));
return entityRichInfo;
}
private ChatAggRichConfigResp fillChatAggRichConfig(
DataSetSchema modelSchema, ChatConfigResp chatConfigResp) {
private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema,
ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {
return null;
}
@@ -284,46 +266,32 @@ public class ConfigServiceImpl implements ConfigService {
chatAggRichConfig.setKnowledgeInfos(
fillKnowledgeBizName(chatAggConfig.getKnowledgeInfos(), modelSchema));
chatAggRichConfig.setGlobalKnowledgeConfig(chatAggConfig.getGlobalKnowledgeConfig());
chatAggRichConfig.setChatDefaultConfig(
fetchDefaultConfig(
chatAggRichConfig.setChatDefaultConfig(fetchDefaultConfig(
chatAggConfig.getChatDefaultConfig(), modelSchema, itemVisibilityInfo));
return chatAggRichConfig;
}
private ChatDefaultRichConfigResp fetchDefaultConfig(
ChatDefaultConfigReq chatDefaultConfig,
DataSetSchema modelSchema,
ItemVisibilityInfo itemVisibilityInfo) {
private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig,
DataSetSchema modelSchema, ItemVisibilityInfo itemVisibilityInfo) {
ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp();
if (Objects.isNull(chatDefaultConfig)) {
return defaultRichConfig;
}
BeanUtils.copyProperties(chatDefaultConfig, defaultRichConfig);
Map<Long, SchemaElement> dimIdAndRespPair =
modelSchema.getDimensions().stream()
.collect(
Collectors.toMap(
SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
Map<Long, SchemaElement> metricIdAndRespPair =
modelSchema.getMetrics().stream()
.collect(
Collectors.toMap(
SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
Map<Long, SchemaElement> metricIdAndRespPair = modelSchema.getMetrics().stream().collect(
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
List<SchemaElement> dimensions = new ArrayList<>();
List<SchemaElement> metrics = new ArrayList<>();
if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensionIds())) {
chatDefaultConfig.getDimensionIds().stream()
.filter(
dimId ->
dimIdAndRespPair.containsKey(dimId)
&& itemVisibilityInfo
.getWhiteDimIdList()
.contains(dimId))
.forEach(
dimId -> {
.filter(dimId -> dimIdAndRespPair.containsKey(dimId)
&& itemVisibilityInfo.getWhiteDimIdList().contains(dimId))
.forEach(dimId -> {
SchemaElement dimSchemaResp = dimIdAndRespPair.get(dimId);
if (Objects.nonNull(dimSchemaResp)) {
SchemaElement dimSchema = new SchemaElement();
@@ -335,14 +303,9 @@ public class ConfigServiceImpl implements ConfigService {
if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetricIds())) {
chatDefaultConfig.getMetricIds().stream()
.filter(
metricId ->
metricIdAndRespPair.containsKey(metricId)
&& itemVisibilityInfo
.getWhiteMetricIdList()
.contains(metricId))
.forEach(
metricId -> {
.filter(metricId -> metricIdAndRespPair.containsKey(metricId)
&& itemVisibilityInfo.getWhiteMetricIdList().contains(metricId))
.forEach(metricId -> {
SchemaElement metricSchemaResp = metricIdAndRespPair.get(metricId);
if (Objects.nonNull(metricSchemaResp)) {
SchemaElement metricSchema = new SchemaElement();
@@ -357,22 +320,16 @@ public class ConfigServiceImpl implements ConfigService {
return defaultRichConfig;
}
private List<KnowledgeInfoReq> fillKnowledgeBizName(
List<KnowledgeInfoReq> knowledgeInfos, DataSetSchema modelSchema) {
private List<KnowledgeInfoReq> fillKnowledgeBizName(List<KnowledgeInfoReq> knowledgeInfos,
DataSetSchema modelSchema) {
if (CollectionUtils.isEmpty(knowledgeInfos)) {
return new ArrayList<>();
}
Map<Long, SchemaElement> dimIdAndRespPair =
modelSchema.getDimensions().stream()
.collect(
Collectors.toMap(
SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
knowledgeInfos.stream()
.forEach(
knowledgeInfo -> {
Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
knowledgeInfos.stream().forEach(knowledgeInfo -> {
if (Objects.nonNull(knowledgeInfo)) {
SchemaElement dimSchemaResp =
dimIdAndRespPair.get(knowledgeInfo.getItemId());
SchemaElement dimSchemaResp = dimIdAndRespPair.get(knowledgeInfo.getItemId());
if (Objects.nonNull(dimSchemaResp)) {
knowledgeInfo.setBizName(dimSchemaResp.getBizName());
}

View File

@@ -25,11 +25,14 @@ import java.util.List;
@Service
public class MemoryServiceImpl implements MemoryService {
@Autowired private ChatMemoryRepository chatMemoryRepository;
@Autowired
private ChatMemoryRepository chatMemoryRepository;
@Autowired private ExemplarService exemplarService;
@Autowired
private ExemplarService exemplarService;
@Autowired private EmbeddingConfig embeddingConfig;
@Autowired
private EmbeddingConfig embeddingConfig;
@Override
public void createMemory(ChatMemoryDO memory) {
@@ -85,20 +88,18 @@ public class MemoryServiceImpl implements MemoryService {
queryWrapper.lambda().eq(ChatMemoryDO::getStatus, chatMemoryFilter.getStatus());
}
if (chatMemoryFilter.getHumanReviewRet() != null) {
queryWrapper
.lambda()
.eq(ChatMemoryDO::getHumanReviewRet, chatMemoryFilter.getHumanReviewRet());
queryWrapper.lambda().eq(ChatMemoryDO::getHumanReviewRet,
chatMemoryFilter.getHumanReviewRet());
}
if (chatMemoryFilter.getLlmReviewRet() != null) {
queryWrapper
.lambda()
.eq(ChatMemoryDO::getLlmReviewRet, chatMemoryFilter.getLlmReviewRet());
queryWrapper.lambda().eq(ChatMemoryDO::getLlmReviewRet,
chatMemoryFilter.getLlmReviewRet());
}
if (StringUtils.isBlank(chatMemoryFilter.getOrderCondition())) {
queryWrapper.orderByDesc("id");
} else {
queryWrapper.orderBy(
true, chatMemoryFilter.isAsc(), chatMemoryFilter.getOrderCondition());
queryWrapper.orderBy(true, chatMemoryFilter.isAsc(),
chatMemoryFilter.getOrderCondition());
}
return chatMemoryRepository.getMemories(queryWrapper);
}
@@ -106,9 +107,7 @@ public class MemoryServiceImpl implements MemoryService {
@Override
public List<ChatMemoryDO> getMemoriesForLlmReview() {
QueryWrapper<ChatMemoryDO> queryWrapper = new QueryWrapper<>();
queryWrapper
.lambda()
.eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING)
queryWrapper.lambda().eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING)
.isNull(ChatMemoryDO::getLlmReviewRet);
return chatMemoryRepository.getMemories(queryWrapper);
}
@@ -116,26 +115,18 @@ public class MemoryServiceImpl implements MemoryService {
@Override
public void enableMemory(ChatMemoryDO memory) {
memory.setStatus(MemoryStatus.ENABLED);
exemplarService.storeExemplar(
embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
Text2SQLExemplar.builder()
.question(memory.getQuestion())
.sideInfo(memory.getSideInfo())
.dbSchema(memory.getDbSchema())
.sql(memory.getS2sql())
.build());
exemplarService.storeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
Text2SQLExemplar.builder().question(memory.getQuestion())
.sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
.sql(memory.getS2sql()).build());
}
@Override
public void disableMemory(ChatMemoryDO memory) {
memory.setStatus(MemoryStatus.DISABLED);
exemplarService.removeExemplar(
embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
Text2SQLExemplar.builder()
.question(memory.getQuestion())
.sideInfo(memory.getSideInfo())
.dbSchema(memory.getDbSchema())
.sql(memory.getS2sql())
.build());
exemplarService.removeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
Text2SQLExemplar.builder().question(memory.getQuestion())
.sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
.sql(memory.getS2sql()).build());
}
}

View File

@@ -36,8 +36,8 @@ public class PluginServiceImpl implements PluginService {
private ApplicationEventPublisher publisher;
public PluginServiceImpl(
PluginRepository pluginRepository, ApplicationEventPublisher publisher) {
public PluginServiceImpl(PluginRepository pluginRepository,
ApplicationEventPublisher publisher) {
this.pluginRepository = pluginRepository;
this.publisher = publisher;
}
@@ -110,17 +110,10 @@ public class PluginServiceImpl implements PluginService {
}
List<PluginDO> pluginDOS = pluginRepository.query(queryWrapper);
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
pluginDOS =
pluginDOS.stream()
.filter(
pluginDO ->
pluginDO.getPattern()
.contains(pluginQueryReq.getPattern())
pluginDOS = pluginDOS.stream()
.filter(pluginDO -> pluginDO.getPattern().contains(pluginQueryReq.getPattern())
|| (pluginDO.getName() != null
&& pluginDO.getName()
.contains(
pluginQueryReq
.getPattern())))
&& pluginDO.getName().contains(pluginQueryReq.getPattern())))
.collect(Collectors.toList());
}
return convertList(pluginDOS);
@@ -129,16 +122,13 @@ public class PluginServiceImpl implements PluginService {
@Override
public Optional<ChatPlugin> getPluginByName(String name) {
log.info("name:{}", name);
return getPluginList().stream()
.filter(
plugin -> {
return getPluginList().stream().filter(plugin -> {
PluginParseConfig functionCallConfig = getPluginParseConfig(plugin);
if (functionCallConfig == null) {
return false;
}
return functionCallConfig.getName().equalsIgnoreCase(name);
})
.findFirst();
}).findFirst();
}
private PluginParseConfig getPluginParseConfig(ChatPlugin plugin) {
@@ -166,26 +156,17 @@ public class PluginServiceImpl implements PluginService {
public Map<String, ChatPlugin> getNameToPlugin() {
List<ChatPlugin> pluginList = getPluginList();
return pluginList.stream()
.filter(
plugin -> {
return pluginList.stream().filter(plugin -> {
PluginParseConfig functionCallConfig = getPluginParseConfig(plugin);
if (functionCallConfig == null) {
return false;
}
return true;
})
.collect(
Collectors.toMap(
a -> {
}).collect(Collectors.toMap(a -> {
PluginParseConfig functionCallConfig =
JsonUtil.toObject(
a.getParseModeConfig(),
PluginParseConfig.class);
JsonUtil.toObject(a.getParseModeConfig(), PluginParseConfig.class);
return functionCallConfig.getName();
},
a -> a,
(k1, k2) -> k1));
}, a -> a, (k1, k2) -> k1));
}
// todo
@@ -197,10 +178,8 @@ public class PluginServiceImpl implements PluginService {
ChatPlugin plugin = new ChatPlugin();
BeanUtils.copyProperties(pluginDO, plugin);
if (StringUtils.isNotBlank(pluginDO.getDataSet())) {
plugin.setDataSetList(
Arrays.stream(pluginDO.getDataSet().split(","))
.map(Long::parseLong)
.collect(Collectors.toList()));
plugin.setDataSetList(Arrays.stream(pluginDO.getDataSet().split(","))
.map(Long::parseLong).collect(Collectors.toList()));
}
return plugin;
}

View File

@@ -14,7 +14,8 @@ import java.util.List;
@Slf4j
public class StatisticsServiceImpl implements StatisticsService {
@Autowired private StatisticsMapper statisticsMapper;
@Autowired
private StatisticsMapper statisticsMapper;
@Async
@Override

View File

@@ -37,9 +37,7 @@ public class ChatConfigHelper {
ChatConfig chatConfig = new ChatConfig();
BeanUtils.copyProperties(extendBaseCmd, chatConfig);
RecordInfo recordInfo = new RecordInfo();
String creator =
(Objects.isNull(user) || StringUtils.isEmpty(user.getName()))
? ADMIN_LOWER
String creator = (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) ? ADMIN_LOWER
: user.getName();
recordInfo.createdBy(creator);
chatConfig.setRecordInfo(recordInfo);
@@ -52,8 +50,7 @@ public class ChatConfigHelper {
BeanUtils.copyProperties(extendEditCmd, chatConfig);
RecordInfo recordInfo = new RecordInfo();
String user =
(Objects.isNull(facadeUser) || StringUtils.isEmpty(facadeUser.getName()))
String user = (Objects.isNull(facadeUser) || StringUtils.isEmpty(facadeUser.getName()))
? ADMIN_LOWER
: facadeUser.getName();
recordInfo.updatedBy(user);
@@ -65,8 +62,7 @@ public class ChatConfigHelper {
if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getDimensions())) {
return new ArrayList<>();
}
Map<Long, List<SchemaElement>> dimIdAndDescPair =
modelSchema.getDimensions().stream()
Map<Long, List<SchemaElement>> dimIdAndDescPair = modelSchema.getDimensions().stream()
.collect(Collectors.groupingBy(SchemaElement::getId));
return new ArrayList<>(dimIdAndDescPair.keySet());
}
@@ -75,8 +71,7 @@ public class ChatConfigHelper {
if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getMetrics())) {
return new ArrayList<>();
}
Map<Long, List<SchemaElement>> metricIdAndDescPair =
modelSchema.getMetrics().stream()
Map<Long, List<SchemaElement>> metricIdAndDescPair = modelSchema.getMetrics().stream()
.collect(Collectors.groupingBy(SchemaElement::getId));
return new ArrayList<>(metricIdAndDescPair.keySet());
}
@@ -87,8 +82,8 @@ public class ChatConfigHelper {
chatConfigDO.setChatAggConfig(JsonUtil.toString(chatConfig.getChatAggConfig()));
chatConfigDO.setChatDetailConfig(JsonUtil.toString(chatConfig.getChatDetailConfig()));
chatConfigDO.setRecommendedQuestions(
JsonUtil.toString(chatConfig.getRecommendedQuestions()));
chatConfigDO
.setRecommendedQuestions(JsonUtil.toString(chatConfig.getRecommendedQuestions()));
if (Objects.isNull(chatConfig.getStatus())) {
chatConfigDO.setStatus(null);
@@ -118,9 +113,8 @@ public class ChatConfigHelper {
JsonUtil.toObject(chatConfigDO.getChatDetailConfig(), ChatDetailConfigReq.class));
chatConfigDescriptor.setChatAggConfig(
JsonUtil.toObject(chatConfigDO.getChatAggConfig(), ChatAggConfigReq.class));
chatConfigDescriptor.setRecommendedQuestions(
JsonUtil.toList(
chatConfigDO.getRecommendedQuestions(), RecommendedQuestionReq.class));
chatConfigDescriptor.setRecommendedQuestions(JsonUtil
.toList(chatConfigDO.getRecommendedQuestions(), RecommendedQuestionReq.class));
chatConfigDescriptor.setStatusEnum(StatusEnum.of(chatConfigDO.getStatus()));
chatConfigDescriptor.setCreatedBy(chatConfigDO.getCreatedBy());

View File

@@ -51,15 +51,13 @@ public class ComponentFactory {
}
private static <T> List<T> init(Class<T> factoryType, List list) {
list.addAll(
SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader()));
list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
Thread.currentThread().getContextClassLoader()));
return list;
}
private static <T> T init(Class<T> factoryType) {
return SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader())
.get(0);
return SpringFactoriesLoader
.loadFactories(factoryType, Thread.currentThread().getContextClassLoader()).get(0);
}
}

View File

@@ -8,8 +8,8 @@ import java.util.Map;
public class ResultFormatter {
public static String transform2TextNew(
List<QueryColumn> queryColumns, List<Map<String, Object>> queryResults) {
public static String transform2TextNew(List<QueryColumn> queryColumns,
List<Map<String, Object>> queryResults) {
if (CollectionUtils.isEmpty(queryColumns)) {
return "";
}

View File

@@ -24,8 +24,7 @@ public class LoadRemoveService {
}
List<String> resultList = new ArrayList<>(value);
if (StringUtils.isNotBlank(mapperRemoveNaturePrefix)) {
resultList.removeIf(
nature -> {
resultList.removeIf(nature -> {
if (Objects.isNull(nature)) {
return false;
}

View File

@@ -253,19 +253,8 @@ public abstract class BaseNode<V> implements Comparable<BaseNode> {
@Override
public String toString() {
return "BaseNode{"
+ "child="
+ Arrays.toString(child)
+ ", status="
+ status
+ ", c="
+ c
+ ", value="
+ value
+ ", prefix='"
+ prefix
+ '\''
+ '}';
return "BaseNode{" + "child=" + Arrays.toString(child) + ", status=" + status + ", c=" + c
+ ", value=" + value + ", prefix='" + prefix + '\'' + '}';
}
public void walkNode(Set<Map.Entry<String, V>> entrySet) {

View File

@@ -34,13 +34,8 @@ public class CoreDictionary {
if (!load(PATH)) {
throw new IllegalArgumentException("核心词典" + PATH + "加载失败");
} else {
Predefine.logger.info(
PATH
+ "加载成功,"
+ trie.size()
+ "个词条,耗时"
+ (System.currentTimeMillis() - start)
+ "ms");
Predefine.logger.info(PATH + "加载成功," + trie.size() + "个词条,耗时"
+ (System.currentTimeMillis() - start) + "ms");
}
}
@@ -77,22 +72,14 @@ public class CoreDictionary {
map.put(param[0], attribute);
totalFrequency += attribute.totalFrequency;
}
Predefine.logger.info(
"核心词典读入词条"
+ map.size()
+ " 全部频次"
+ totalFrequency
+ ",耗时"
+ (System.currentTimeMillis() - start)
+ "ms");
Predefine.logger.info("核心词典读入词条" + map.size() + " 全部频次" + totalFrequency + ",耗时"
+ (System.currentTimeMillis() - start) + "ms");
br.close();
trie.build(map);
Predefine.logger.info("核心词典加载成功:" + trie.size() + "个词条,下面将写入缓存……");
try {
DataOutputStream out =
new DataOutputStream(
new BufferedOutputStream(
IOUtil.newOutputStream(path + Predefine.BIN_EXT)));
DataOutputStream out = new DataOutputStream(
new BufferedOutputStream(IOUtil.newOutputStream(path + Predefine.BIN_EXT)));
Collection<Attribute> attributeList = map.values();
out.writeInt(attributeList.size());
for (Attribute attribute : attributeList) {
@@ -278,10 +265,7 @@ public class CoreDictionary {
}
return attribute;
} catch (Exception e) {
Predefine.logger.warning(
"使用字符串"
+ natureWithFrequency
+ "创建词条属性失败!"
Predefine.logger.warning("使用字符串" + natureWithFrequency + "创建词条属性失败!"
+ TextUtility.exceptionToString(e));
return null;
}
@@ -409,9 +393,7 @@ public class CoreDictionary {
if (originals == null || originals.length == 0) {
return null;
}
return Arrays.stream(originals)
.filter(o -> o != null)
.distinct()
return Arrays.stream(originals).filter(o -> o != null).distinct()
.collect(Collectors.toList());
}
}

View File

@@ -47,8 +47,7 @@ public abstract class WordBasedSegment extends Segment {
}
vertex = (Vertex) var1.next();
} while (!vertex.realWord.equals("")
&& !vertex.realWord.equals("")
} while (!vertex.realWord.equals("") && !vertex.realWord.equals("")
&& !vertex.realWord.equals("-"));
vertex.confirmNature(Nature.w);
@@ -66,8 +65,7 @@ public abstract class WordBasedSegment extends Segment {
if (currentNature == Nature.nx
&& (next.hasNature(Nature.q) || next.hasNature(Nature.n))) {
String[] param = current.realWord.split("-", 1);
if (param.length == 2
&& TextUtility.isAllNum(param[0])
if (param.length == 2 && TextUtility.isAllNum(param[0])
&& TextUtility.isAllNum(param[1])) {
current = current.copy();
current.realWord = param[0];
@@ -112,10 +110,8 @@ public abstract class WordBasedSegment extends Segment {
current.confirmNature(Nature.m, true);
} else if (current.realWord.length() > 1) {
char last = current.realWord.charAt(current.realWord.length() - 1);
current =
Vertex.newNumberInstance(
current.realWord.substring(
0, current.realWord.length() - 1));
current = Vertex.newNumberInstance(
current.realWord.substring(0, current.realWord.length() - 1));
listIterator.previous();
listIterator.previous();
listIterator.set(current);
@@ -162,9 +158,7 @@ public abstract class WordBasedSegment extends Segment {
charTypeArray[i] = CharType.get(c);
if (c == '.' && i < charArray.length - 1 && CharType.get(charArray[i + 1]) == 9) {
charTypeArray[i] = 9;
} else if (c == '.'
&& i < charArray.length - 1
&& charArray[i + 1] >= '0'
} else if (c == '.' && i < charArray.length - 1 && charArray[i + 1] >= '0'
&& charArray[i + 1] <= '9') {
charTypeArray[i] = 5;
} else if (charTypeArray[i] == 8) {
@@ -252,21 +246,16 @@ public abstract class WordBasedSegment extends Segment {
DoubleArrayTrie.Searcher searcher = CoreDictionary.trie.getSearcher(charArray, 0);
while (searcher.next()) {
wordNetStorage.add(
searcher.begin + 1,
new Vertex(
new String(charArray, searcher.begin, searcher.length),
(CoreDictionary.Attribute) searcher.value,
searcher.index));
wordNetStorage.add(searcher.begin + 1,
new Vertex(new String(charArray, searcher.begin, searcher.length),
(CoreDictionary.Attribute) searcher.value, searcher.index));
}
if (this.config.forceCustomDictionary) {
this.customDictionary.parseText(
charArray,
this.customDictionary.parseText(charArray,
new AhoCorasickDoubleArrayTrie.IHit<CoreDictionary.Attribute>() {
public void hit(int begin, int end, CoreDictionary.Attribute value) {
wordNetStorage.add(
begin + 1,
wordNetStorage.add(begin + 1,
new Vertex(new String(charArray, begin, end - begin), value));
}
});
@@ -279,11 +268,9 @@ public abstract class WordBasedSegment extends Segment {
while (i < vertexes.length) {
if (vertexes[i].isEmpty()) {
int j;
for (j = i + 1;
j < vertexes.length - 1
&& (vertexes[j].isEmpty()
|| CharType.get(charArray[j - 1]) == 11);
++j) {}
for (j = i + 1; j < vertexes.length - 1 && (vertexes[j].isEmpty()
|| CharType.get(charArray[j - 1]) == 11); ++j) {
}
wordNetStorage.add(i, Segment.quickAtomSegment(charArray, i - 1, j - 1));
i = j;
@@ -310,10 +297,8 @@ public abstract class WordBasedSegment extends Segment {
addTerms(termList, vertex, line - 1);
termMain.offset = line - 1;
if (vertex.realWord.length() > 2) {
label43:
for (int currentLine = line;
currentLine < line + vertex.realWord.length();
++currentLine) {
label43: for (int currentLine = line; currentLine < line
+ vertex.realWord.length(); ++currentLine) {
Iterator iterator = wordNetAll.descendingIterator(currentLine);
while (true) {
@@ -327,8 +312,8 @@ public abstract class WordBasedSegment extends Segment {
&& smallVertex.realWord.length() < this.config.indexMode);
if (smallVertex != vertex
&& currentLine + smallVertex.realWord.length()
<= line + vertex.realWord.length()) {
&& currentLine + smallVertex.realWord.length() <= line
+ vertex.realWord.length()) {
listIterator.add(smallVertex);
// Term termSub = convert(smallVertex);
// termSub.offset = currentLine - 1;
@@ -346,8 +331,8 @@ public abstract class WordBasedSegment extends Segment {
}
protected static void speechTagging(List<Vertex> vertexList) {
Viterbi.compute(
vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
Viterbi.compute(vertexList,
CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
}
protected void addTerms(List<Term> terms, Vertex vertex, int offset) {

View File

@@ -42,18 +42,12 @@ public class Term {
}
// todo opt
/*
String wordOri = word.toLowerCase();
CoreDictionary.Attribute attribute = getDynamicCustomDictionary().get(wordOri);
if (attribute == null) {
attribute = CoreDictionary.get(wordOri);
if (attribute == null) {
attribute = CustomDictionary.get(wordOri);
}
}
if (attribute != null && nature != null && attribute.hasNature(nature)) {
return attribute.getNatureFrequency(nature);
}
return attribute == null ? 0 : attribute.totalFrequency;
* String wordOri = word.toLowerCase(); CoreDictionary.Attribute attribute =
* getDynamicCustomDictionary().get(wordOri); if (attribute == null) { attribute =
* CoreDictionary.get(wordOri); if (attribute == null) { attribute =
* CustomDictionary.get(wordOri); } } if (attribute != null && nature != null &&
* attribute.hasNature(nature)) { return attribute.getNatureFrequency(nature); } return
* attribute == null ? 0 : attribute.totalFrequency;
*/
return 0;
}

View File

@@ -51,19 +51,18 @@ public class Configuration {
public static SqlValidator.Config getValidatorConfig(EngineType engineType) {
SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType);
return SqlValidator.Config.DEFAULT
.withConformance(sqlDialect.getConformance())
return SqlValidator.Config.DEFAULT.withConformance(sqlDialect.getConformance())
.withDefaultNullCollation(config.defaultNullCollation())
.withLenientOperatorLookup(true);
}
static {
configProperties.put(
CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString());
configProperties.put(
CalciteConnectionProperty.UNQUOTED_CASING.camelName(), Casing.UNCHANGED.toString());
configProperties.put(
CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.TO_LOWER.toString());
configProperties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(),
Boolean.TRUE.toString());
configProperties.put(CalciteConnectionProperty.UNQUOTED_CASING.camelName(),
Casing.UNCHANGED.toString());
configProperties.put(CalciteConnectionProperty.QUOTED_CASING.camelName(),
Casing.TO_LOWER.toString());
}
public static SqlParser.Config getParserConfig(EngineType engineType) {
@@ -76,15 +75,10 @@ public class Configuration {
parserConfig.setQuotedCasing(config.quotedCasing());
parserConfig.setConformance(config.conformance());
parserConfig.setLex(Lex.BIG_QUERY);
parserConfig
.setParserFactory(SqlParserImpl.FACTORY)
.setCaseSensitive(false)
.setIdentifierMaxLength(Integer.MAX_VALUE)
.setQuoting(Quoting.BACK_TICK)
.setQuoting(Quoting.SINGLE_QUOTE)
.setQuotedCasing(Casing.TO_UPPER)
.setUnquotedCasing(Casing.TO_UPPER)
.setConformance(sqlDialect.getConformance())
parserConfig.setParserFactory(SqlParserImpl.FACTORY).setCaseSensitive(false)
.setIdentifierMaxLength(Integer.MAX_VALUE).setQuoting(Quoting.BACK_TICK)
.setQuoting(Quoting.SINGLE_QUOTE).setQuotedCasing(Casing.TO_UPPER)
.setUnquotedCasing(Casing.TO_UPPER).setConformance(sqlDialect.getConformance())
.setLex(Lex.BIG_QUERY);
parserConfig = parserConfig.setQuotedCasing(Casing.UNCHANGED);
parserConfig = parserConfig.setUnquotedCasing(Casing.UNCHANGED);
@@ -96,61 +90,39 @@ public class Configuration {
tables.add(SqlStdOperatorTable.instance());
SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables);
// operatorTable.
Prepare.CatalogReader catalogReader =
new CalciteCatalogReader(
rootSchema,
Collections.singletonList(rootSchema.getName()),
typeFactory,
config);
return SqlValidatorUtil.newValidator(
operatorTable,
catalogReader,
typeFactory,
Prepare.CatalogReader catalogReader = new CalciteCatalogReader(rootSchema,
Collections.singletonList(rootSchema.getName()), typeFactory, config);
return SqlValidatorUtil.newValidator(operatorTable, catalogReader, typeFactory,
Configuration.getValidatorConfig(engineType));
}
public static SqlValidatorWithHints getSqlValidatorWithHints(
CalciteSchema rootSchema, EngineType engineTyp) {
return new SqlAdvisorValidator(
SqlStdOperatorTable.instance(),
new CalciteCatalogReader(
rootSchema,
Collections.singletonList(rootSchema.getName()),
typeFactory,
config),
typeFactory,
SqlValidator.Config.DEFAULT);
public static SqlValidatorWithHints getSqlValidatorWithHints(CalciteSchema rootSchema,
EngineType engineTyp) {
return new SqlAdvisorValidator(SqlStdOperatorTable.instance(),
new CalciteCatalogReader(rootSchema,
Collections.singletonList(rootSchema.getName()), typeFactory, config),
typeFactory, SqlValidator.Config.DEFAULT);
}
public static SqlToRelConverter.Config getConverterConfig() {
HintStrategyTable strategies = HintStrategyTable.builder().build();
return SqlToRelConverter.config()
.withHintStrategyTable(strategies)
.withTrimUnusedFields(true)
.withExpand(true)
return SqlToRelConverter.config().withHintStrategyTable(strategies)
.withTrimUnusedFields(true).withExpand(true)
.addRelBuilderConfigTransform(c -> c.withSimplify(false));
}
public static SqlToRelConverter getSqlToRelConverter(
SqlValidatorScope scope,
SqlValidator sqlValidator,
RelOptPlanner relOptPlanner,
EngineType engineType) {
public static SqlToRelConverter getSqlToRelConverter(SqlValidatorScope scope,
SqlValidator sqlValidator, RelOptPlanner relOptPlanner, EngineType engineType) {
RexBuilder rexBuilder = new RexBuilder(typeFactory);
RelOptCluster cluster = RelOptCluster.create(relOptPlanner, rexBuilder);
FrameworkConfig fromworkConfig =
Frameworks.newConfigBuilder()
.parserConfig(getParserConfig(engineType))
Frameworks.newConfigBuilder().parserConfig(getParserConfig(engineType))
.defaultSchema(
scope.getValidator().getCatalogReader().getRootSchema().plus())
.build();
return new SqlToRelConverter(
new ViewExpanderImpl(),
sqlValidator,
(CatalogReader) scope.getValidator().getCatalogReader(),
cluster,
fromworkConfig.getConvertletTable(),
getConverterConfig());
return new SqlToRelConverter(new ViewExpanderImpl(), sqlValidator,
(CatalogReader) scope.getValidator().getCatalogReader(), cluster,
fromworkConfig.getConvertletTable(), getConverterConfig());
}
public static SqlAdvisor getSqlAdvisor(SqlValidatorWithHints validator, EngineType engineType) {
@@ -159,15 +131,10 @@ public class Configuration {
public static SqlWriterConfig getSqlWriterConfig(EngineType engineType) {
SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType);
SqlWriterConfig config =
SqlPrettyWriter.config()
.withDialect(sqlDialect)
.withKeywordsLowerCase(false)
.withClauseEndsLine(true)
.withAlwaysUseParentheses(false)
.withSelectListItemsOnSeparateLines(false)
.withUpdateSetListNewline(false)
.withIndentation(0);
SqlWriterConfig config = SqlPrettyWriter.config().withDialect(sqlDialect)
.withKeywordsLowerCase(false).withClauseEndsLine(true)
.withAlwaysUseParentheses(false).withSelectListItemsOnSeparateLines(false)
.withUpdateSetListNewline(false).withIndentation(0);
if (EngineType.MYSQL.equals(engineType)) {
// no backticks around function name
config = config.withQuoteAllIdentifiers(false);

View File

@@ -17,8 +17,8 @@ public class SemanticSqlDialect extends SqlDialect {
super(context);
}
public static void unparseFetchUsingAnsi(
SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) {
public static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode offset,
@Nullable SqlNode fetch) {
Preconditions.checkArgument(fetch != null || offset != null);
SqlWriter.Frame fetchFrame;
writer.newlineAndIndent();
@@ -74,11 +74,11 @@ public class SemanticSqlDialect extends SqlDialect {
return true;
}
public void unparseSqlIntervalLiteral(
SqlWriter writer, SqlIntervalLiteral literal, int leftPrec, int rightPrec) {}
public void unparseSqlIntervalLiteral(SqlWriter writer, SqlIntervalLiteral literal,
int leftPrec, int rightPrec) {}
public void unparseOffsetFetch(
SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) {
public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset,
@Nullable SqlNode fetch) {
unparseFetchUsingAnsi(writer, offset, fetch);
}
}

View File

@@ -13,22 +13,14 @@ import java.util.Objects;
public class SqlDialectFactory {
public static final Context DEFAULT_CONTEXT =
SqlDialect.EMPTY_CONTEXT
.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
.withLiteralQuoteString("'")
.withLiteralEscapedQuoteString("''")
.withIdentifierQuoteString("`")
.withUnquotedCasing(Casing.UNCHANGED)
.withQuotedCasing(Casing.UNCHANGED)
.withCaseSensitive(false);
public static final Context POSTGRESQL_CONTEXT =
SqlDialect.EMPTY_CONTEXT
.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
.withLiteralQuoteString("'")
.withLiteralEscapedQuoteString("''")
.withUnquotedCasing(Casing.UNCHANGED)
.withQuotedCasing(Casing.UNCHANGED)
.withCaseSensitive(false);
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
.withLiteralQuoteString("'").withLiteralEscapedQuoteString("''")
.withIdentifierQuoteString("`").withUnquotedCasing(Casing.UNCHANGED)
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false);
public static final Context POSTGRESQL_CONTEXT = SqlDialect.EMPTY_CONTEXT
.withDatabaseProduct(DatabaseProduct.BIG_QUERY).withLiteralQuoteString("'")
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false);
private static Map<EngineType, SemanticSqlDialect> sqlDialectMap;
static {

View File

@@ -20,12 +20,8 @@ import java.util.List;
@Slf4j
public class SqlMergeWithUtils {
public static String mergeWith(
EngineType engineType,
String sql,
List<String> parentSqlList,
List<String> parentWithNameList)
throws SqlParseException {
public static String mergeWith(EngineType engineType, String sql, List<String> parentSqlList,
List<String> parentWithNameList) throws SqlParseException {
SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
// Parse the main SQL statement
@@ -45,14 +41,12 @@ public class SqlMergeWithUtils {
SqlNode sqlNode2 = parser.parseQuery();
// Create a new WITH item for parentWithName without quotes
SqlWithItem withItem =
new SqlWithItem(
SqlParserPos.ZERO,
new SqlIdentifier(
parentWithName, SqlParserPos.ZERO), // false to avoid quotes
null,
sqlNode2,
SqlLiteral.createBoolean(false, SqlParserPos.ZERO));
SqlWithItem withItem = new SqlWithItem(SqlParserPos.ZERO,
new SqlIdentifier(parentWithName, SqlParserPos.ZERO), // false
// to
// avoid
// quotes
null, sqlNode2, SqlLiteral.createBoolean(false, SqlParserPos.ZERO));
// Add the new WITH item to the list
withItemList.add(withItem);
@@ -66,11 +60,8 @@ public class SqlMergeWithUtils {
}
// Create a new SqlWith node
SqlWith finalSqlNode =
new SqlWith(
SqlParserPos.ZERO,
new SqlNodeList(withItemList, SqlParserPos.ZERO),
sqlNode1);
SqlWith finalSqlNode = new SqlWith(SqlParserPos.ZERO,
new SqlNodeList(withItemList, SqlParserPos.ZERO), sqlNode1);
// Custom SqlPrettyWriter configuration to avoid quoting identifiers
SqlWriterConfig config = Configuration.getSqlWriterConfig(engineType);
// Pretty print the final SQL

View File

@@ -45,9 +45,7 @@ public class SqlParseUtils {
sqlParserInfo.setAllFields(
sqlParserInfo.getAllFields().stream().distinct().collect(Collectors.toList()));
sqlParserInfo.setSelectFields(
sqlParserInfo.getSelectFields().stream()
.distinct()
sqlParserInfo.setSelectFields(sqlParserInfo.getSelectFields().stream().distinct()
.collect(Collectors.toList()));
return sqlParserInfo;
@@ -108,10 +106,7 @@ public class SqlParseUtils {
SqlSelect sqlSelect = (SqlSelect) select;
SqlNodeList selectList = sqlSelect.getSelectList();
selectList
.getList()
.forEach(
list -> {
selectList.getList().forEach(list -> {
Set<String> selectFields = handlerField(list);
sqlParserInfo.getSelectFields().addAll(selectFields);
});
@@ -129,11 +124,7 @@ public class SqlParseUtils {
results.addAll(formFields);
}
sqlSelect
.getSelectList()
.getList()
.forEach(
list -> {
sqlSelect.getSelectList().getList().forEach(list -> {
Set<String> selectFields = handlerField(list);
results.addAll(selectFields);
});
@@ -148,8 +139,7 @@ public class SqlParseUtils {
}
SqlNodeList group = sqlSelect.getGroup();
if (group != null) {
group.forEach(
groupField -> {
group.forEach(groupField -> {
Set<String> groupByFields = handlerField(groupField);
results.addAll(groupByFields);
});
@@ -213,10 +203,7 @@ public class SqlParseUtils {
}
}
if (field instanceof SqlNodeList) {
((SqlNodeList) field)
.getList()
.forEach(
node -> {
((SqlNodeList) field).getList().forEach(node -> {
fields.addAll(handlerField(node));
});
}
@@ -243,12 +230,9 @@ public class SqlParseUtils {
SqlIdentifier sqlIdentifier = (SqlIdentifier) operandList.get(0);
String simple = sqlIdentifier.getSimple();
SqlBasicCall aliasedNode =
new SqlBasicCall(
SqlStdOperatorTable.AS,
new SqlNode[] {
sqlBasicCall,
new SqlIdentifier(simple.toLowerCase(), SqlParserPos.ZERO)
},
new SqlBasicCall(SqlStdOperatorTable.AS,
new SqlNode[] {sqlBasicCall, new SqlIdentifier(
simple.toLowerCase(), SqlParserPos.ZERO)},
SqlParserPos.ZERO);
selectList.set(selectList.indexOf(node), aliasedNode);
}

View File

@@ -11,10 +11,7 @@ public class ViewExpanderImpl implements RelOptTable.ViewExpander {
public ViewExpanderImpl() {}
@Override
public RelRoot expandView(
RelDataType rowType,
String queryString,
List<String> schemaPath,
public RelRoot expandView(RelDataType rowType, String queryString, List<String> schemaPath,
List<String> dataSetPath) {
return null;
}

View File

@@ -20,81 +20,27 @@ import java.util.List;
@Slf4j
public class ChatModelParameterConfig extends ParameterConfig {
public static final Parameter CHAT_MODEL_PROVIDER =
new Parameter(
"s2.chat.model.provider",
OpenAiModelFactory.PROVIDER,
"接口协议",
"",
"list",
"对话模型配置",
getCandidateValues());
public static final Parameter CHAT_MODEL_PROVIDER = new Parameter("s2.chat.model.provider",
OpenAiModelFactory.PROVIDER, "接口协议", "", "list", "对话模型配置", getCandidateValues());
public static final Parameter CHAT_MODEL_BASE_URL =
new Parameter(
"s2.chat.model.base.url",
OpenAiModelFactory.DEFAULT_BASE_URL,
"BaseUrl",
"",
"string",
"对话模型配置",
null,
getBaseUrlDependency());
public static final Parameter CHAT_MODEL_ENDPOINT =
new Parameter(
"s2.chat.model.endpoint",
"llama_2_70b",
"Endpoint",
"",
"string",
"对话模型配置",
null,
getEndpointDependency());
public static final Parameter CHAT_MODEL_API_KEY =
new Parameter(
"s2.chat.model.api.key",
DEMO,
"ApiKey",
"",
"password",
"对话模型配置",
null,
getApiKeyDependency());
public static final Parameter CHAT_MODEL_SECRET_KEY =
new Parameter(
"s2.chat.model.secretKey",
"demo",
"SecretKey",
"",
"password",
"对话模型配置",
null,
getSecretKeyDependency());
new Parameter("s2.chat.model.base.url", OpenAiModelFactory.DEFAULT_BASE_URL, "BaseUrl",
"", "string", "对话模型配置", null, getBaseUrlDependency());
public static final Parameter CHAT_MODEL_ENDPOINT = new Parameter("s2.chat.model.endpoint",
"llama_2_70b", "Endpoint", "", "string", "对话模型配置", null, getEndpointDependency());
public static final Parameter CHAT_MODEL_API_KEY = new Parameter("s2.chat.model.api.key", DEMO,
"ApiKey", "", "password", "对话模型配置", null, getApiKeyDependency());
public static final Parameter CHAT_MODEL_SECRET_KEY = new Parameter("s2.chat.model.secretKey",
"demo", "SecretKey", "", "password", "对话模型配置", null, getSecretKeyDependency());
public static final Parameter CHAT_MODEL_NAME =
new Parameter(
"s2.chat.model.name",
"gpt-4o-mini",
"ModelName",
"",
"string",
"对话模型配置",
null,
getModelNameDependency());
public static final Parameter CHAT_MODEL_NAME = new Parameter("s2.chat.model.name",
"gpt-4o-mini", "ModelName", "", "string", "对话模型配置", null, getModelNameDependency());
public static final Parameter CHAT_MODEL_ENABLE_SEARCH =
new Parameter(
"s2.chat.model.enableSearch",
"false",
"是否启用搜索增强功能设为false表示不启用",
"",
"bool",
"对话模型配置",
null,
getEnableSearchDependency());
new Parameter("s2.chat.model.enableSearch", "false", "是否启用搜索增强功能设为false表示不启用", "",
"bool", "对话模型配置", null, getEnableSearchDependency());
public static final Parameter CHAT_MODEL_TEMPERATURE =
new Parameter(
public static final Parameter CHAT_MODEL_TEMPERATURE = new Parameter(
"s2.chat.model.temperature", "0.0", "Temperature", "", "slider", "对话模型配置");
public static final Parameter CHAT_MODEL_TIMEOUT =
@@ -102,16 +48,9 @@ public class ChatModelParameterConfig extends ParameterConfig {
@Override
public List<Parameter> getSysParameters() {
return Lists.newArrayList(
CHAT_MODEL_PROVIDER,
CHAT_MODEL_BASE_URL,
CHAT_MODEL_ENDPOINT,
CHAT_MODEL_API_KEY,
CHAT_MODEL_SECRET_KEY,
CHAT_MODEL_NAME,
CHAT_MODEL_ENABLE_SEARCH,
CHAT_MODEL_TEMPERATURE,
CHAT_MODEL_TIMEOUT);
return Lists.newArrayList(CHAT_MODEL_PROVIDER, CHAT_MODEL_BASE_URL, CHAT_MODEL_ENDPOINT,
CHAT_MODEL_API_KEY, CHAT_MODEL_SECRET_KEY, CHAT_MODEL_NAME,
CHAT_MODEL_ENABLE_SEARCH, CHAT_MODEL_TEMPERATURE, CHAT_MODEL_TIMEOUT);
}
public ChatModelConfig convert() {
@@ -125,36 +64,24 @@ public class ChatModelParameterConfig extends ParameterConfig {
String secretKey = getParameterValue(CHAT_MODEL_SECRET_KEY);
String enableSearch = getParameterValue(CHAT_MODEL_ENABLE_SEARCH);
return ChatModelConfig.builder()
.provider(chatModelProvider)
.baseUrl(chatModelBaseUrl)
.apiKey(chatModelApiKey)
.modelName(chatModelName)
return ChatModelConfig.builder().provider(chatModelProvider).baseUrl(chatModelBaseUrl)
.apiKey(chatModelApiKey).modelName(chatModelName)
.enableSearch(Boolean.valueOf(enableSearch))
.temperature(Double.valueOf(chatModelTemperature))
.timeOut(Long.valueOf(chatModelTimeout))
.endpoint(endpoint)
.secretKey(secretKey)
.timeOut(Long.valueOf(chatModelTimeout)).endpoint(endpoint).secretKey(secretKey)
.build();
}
private static List<String> getCandidateValues() {
return Lists.newArrayList(
OpenAiModelFactory.PROVIDER,
OllamaModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER,
LocalAiModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER,
return Lists.newArrayList(OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER,
LocalAiModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
AzureModelFactory.PROVIDER);
}
private static List<Parameter.Dependency> getBaseUrlDependency() {
return getDependency(
CHAT_MODEL_PROVIDER.getName(),
getCandidateValues(),
ImmutableMap.of(
OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
return getDependency(CHAT_MODEL_PROVIDER.getName(), getCandidateValues(),
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_BASE_URL,
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_BASE_URL,
QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_BASE_URL,
@@ -164,30 +91,18 @@ public class ChatModelParameterConfig extends ParameterConfig {
}
private static List<Parameter.Dependency> getApiKeyDependency() {
return getDependency(
CHAT_MODEL_PROVIDER.getName(),
Lists.newArrayList(
OpenAiModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER,
LocalAiModelFactory.PROVIDER,
AzureModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER),
ImmutableMap.of(
OpenAiModelFactory.PROVIDER, DEMO,
QianfanModelFactory.PROVIDER, DEMO,
ZhipuModelFactory.PROVIDER, DEMO,
LocalAiModelFactory.PROVIDER, DEMO,
AzureModelFactory.PROVIDER, DEMO,
DashscopeModelFactory.PROVIDER, DEMO));
return getDependency(CHAT_MODEL_PROVIDER.getName(),
Lists.newArrayList(OpenAiModelFactory.PROVIDER, QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER, LocalAiModelFactory.PROVIDER,
AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER),
ImmutableMap.of(OpenAiModelFactory.PROVIDER, DEMO, QianfanModelFactory.PROVIDER,
DEMO, ZhipuModelFactory.PROVIDER, DEMO, LocalAiModelFactory.PROVIDER, DEMO,
AzureModelFactory.PROVIDER, DEMO, DashscopeModelFactory.PROVIDER, DEMO));
}
private static List<Parameter.Dependency> getModelNameDependency() {
return getDependency(
CHAT_MODEL_PROVIDER.getName(),
getCandidateValues(),
ImmutableMap.of(
OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_MODEL_NAME,
return getDependency(CHAT_MODEL_PROVIDER.getName(), getCandidateValues(),
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_MODEL_NAME,
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_MODEL_NAME,
QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_MODEL_NAME,
ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_MODEL_NAME,
@@ -197,23 +112,19 @@ public class ChatModelParameterConfig extends ParameterConfig {
}
private static List<Parameter.Dependency> getEndpointDependency() {
return getDependency(
CHAT_MODEL_PROVIDER.getName(),
Lists.newArrayList(QianfanModelFactory.PROVIDER),
ImmutableMap.of(
QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_ENDPOINT));
return getDependency(CHAT_MODEL_PROVIDER.getName(),
Lists.newArrayList(QianfanModelFactory.PROVIDER), ImmutableMap
.of(QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_ENDPOINT));
}
private static List<Parameter.Dependency> getEnableSearchDependency() {
return getDependency(
CHAT_MODEL_PROVIDER.getName(),
return getDependency(CHAT_MODEL_PROVIDER.getName(),
Lists.newArrayList(DashscopeModelFactory.PROVIDER),
ImmutableMap.of(DashscopeModelFactory.PROVIDER, "false"));
}
private static List<Parameter.Dependency> getSecretKeyDependency() {
return getDependency(
CHAT_MODEL_PROVIDER.getName(),
return getDependency(CHAT_MODEL_PROVIDER.getName(),
Lists.newArrayList(QianfanModelFactory.PROVIDER),
ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO));
}

View File

@@ -22,89 +22,35 @@ import java.util.List;
@Slf4j
public class EmbeddingModelParameterConfig extends ParameterConfig {
public static final Parameter EMBEDDING_MODEL_PROVIDER =
new Parameter(
"s2.embedding.model.provider",
InMemoryModelFactory.PROVIDER,
"接口协议",
"",
"list",
"向量模型配置",
getCandidateValues());
new Parameter("s2.embedding.model.provider", InMemoryModelFactory.PROVIDER, "接口协议", "",
"list", "向量模型配置", getCandidateValues());
public static final Parameter EMBEDDING_MODEL_BASE_URL =
new Parameter(
"s2.embedding.model.base.url",
"",
"BaseUrl",
"",
"string",
"向量模型配置",
null,
getBaseUrlDependency());
new Parameter("s2.embedding.model.base.url", "", "BaseUrl", "", "string", "向量模型配置",
null, getBaseUrlDependency());
public static final Parameter EMBEDDING_MODEL_API_KEY =
new Parameter(
"s2.embedding.model.api.key",
"",
"ApiKey",
"",
"password",
"向量模型配置",
null,
getApiKeyDependency());
new Parameter("s2.embedding.model.api.key", "", "ApiKey", "", "password", "向量模型配置",
null, getApiKeyDependency());
public static final Parameter EMBEDDING_MODEL_SECRET_KEY =
new Parameter(
"s2.embedding.model.secretKey",
"demo",
"SecretKey",
"",
"password",
"向量模型配置",
null,
getSecretKeyDependency());
new Parameter("s2.embedding.model.secretKey", "demo", "SecretKey", "", "password",
"向量模型配置", null, getSecretKeyDependency());
public static final Parameter EMBEDDING_MODEL_NAME =
new Parameter(
"s2.embedding.model.name",
EmbeddingModelConstant.BGE_SMALL_ZH,
"ModelName",
"",
"string",
"向量模型配置",
null,
getModelNameDependency());
new Parameter("s2.embedding.model.name", EmbeddingModelConstant.BGE_SMALL_ZH,
"ModelName", "", "string", "向量模型配置", null, getModelNameDependency());
public static final Parameter EMBEDDING_MODEL_PATH =
new Parameter(
"s2.embedding.model.path",
"",
"模型路径",
"",
"string",
"向量模型配置",
null,
getModelPathDependency());
public static final Parameter EMBEDDING_MODEL_PATH = new Parameter("s2.embedding.model.path",
"", "模型路径", "", "string", "向量模型配置", null, getModelPathDependency());
public static final Parameter EMBEDDING_MODEL_VOCABULARY_PATH =
new Parameter(
"s2.embedding.model.vocabulary.path",
"",
"词汇表路径",
"",
"string",
"向量模型配置",
null,
getModelPathDependency());
new Parameter("s2.embedding.model.vocabulary.path", "", "词汇表路径", "", "string", "向量模型配置",
null, getModelPathDependency());
@Override
public List<Parameter> getSysParameters() {
return Lists.newArrayList(
EMBEDDING_MODEL_PROVIDER,
EMBEDDING_MODEL_BASE_URL,
EMBEDDING_MODEL_API_KEY,
EMBEDDING_MODEL_SECRET_KEY,
EMBEDDING_MODEL_NAME,
EMBEDDING_MODEL_PATH,
EMBEDDING_MODEL_VOCABULARY_PATH);
return Lists.newArrayList(EMBEDDING_MODEL_PROVIDER, EMBEDDING_MODEL_BASE_URL,
EMBEDDING_MODEL_API_KEY, EMBEDDING_MODEL_SECRET_KEY, EMBEDDING_MODEL_NAME,
EMBEDDING_MODEL_PATH, EMBEDDING_MODEL_VOCABULARY_PATH);
}
public EmbeddingModelConfig convert() {
@@ -115,40 +61,24 @@ public class EmbeddingModelParameterConfig extends ParameterConfig {
String modelPath = getParameterValue(EMBEDDING_MODEL_PATH);
String vocabularyPath = getParameterValue(EMBEDDING_MODEL_VOCABULARY_PATH);
String secretKey = getParameterValue(EMBEDDING_MODEL_SECRET_KEY);
return EmbeddingModelConfig.builder()
.provider(provider)
.baseUrl(baseUrl)
.apiKey(apiKey)
.secretKey(secretKey)
.modelName(modelName)
.modelPath(modelPath)
.vocabularyPath(vocabularyPath)
.build();
return EmbeddingModelConfig.builder().provider(provider).baseUrl(baseUrl).apiKey(apiKey)
.secretKey(secretKey).modelName(modelName).modelPath(modelPath)
.vocabularyPath(vocabularyPath).build();
}
private static ArrayList<String> getCandidateValues() {
return Lists.newArrayList(
InMemoryModelFactory.PROVIDER,
OpenAiModelFactory.PROVIDER,
OllamaModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER,
return Lists.newArrayList(InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER,
OllamaModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER,
AzureModelFactory.PROVIDER);
}
private static List<Parameter.Dependency> getBaseUrlDependency() {
return getDependency(
EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(
OpenAiModelFactory.PROVIDER,
OllamaModelFactory.PROVIDER,
AzureModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER),
ImmutableMap.of(
OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER,
AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER),
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_BASE_URL,
AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_BASE_URL,
DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_BASE_URL,
@@ -157,45 +87,27 @@ public class EmbeddingModelParameterConfig extends ParameterConfig {
}
private static List<Parameter.Dependency> getApiKeyDependency() {
return getDependency(
EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(
OpenAiModelFactory.PROVIDER,
AzureModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER,
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(OpenAiModelFactory.PROVIDER, AzureModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER),
ImmutableMap.of(
OpenAiModelFactory.PROVIDER,
DEMO,
AzureModelFactory.PROVIDER,
DEMO,
DashscopeModelFactory.PROVIDER,
DEMO,
QianfanModelFactory.PROVIDER,
DEMO,
ZhipuModelFactory.PROVIDER,
DEMO));
ImmutableMap.of(OpenAiModelFactory.PROVIDER, DEMO, AzureModelFactory.PROVIDER, DEMO,
DashscopeModelFactory.PROVIDER, DEMO, QianfanModelFactory.PROVIDER, DEMO,
ZhipuModelFactory.PROVIDER, DEMO));
}
private static List<Parameter.Dependency> getModelNameDependency() {
return getDependency(
EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(
InMemoryModelFactory.PROVIDER,
OpenAiModelFactory.PROVIDER,
OllamaModelFactory.PROVIDER,
AzureModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER,
QianfanModelFactory.PROVIDER,
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER,
OllamaModelFactory.PROVIDER, AzureModelFactory.PROVIDER,
DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER,
ZhipuModelFactory.PROVIDER),
ImmutableMap.of(
InMemoryModelFactory.PROVIDER, EmbeddingModelConstant.BGE_SMALL_ZH,
ImmutableMap.of(InMemoryModelFactory.PROVIDER, EmbeddingModelConstant.BGE_SMALL_ZH,
OpenAiModelFactory.PROVIDER,
OpenAiModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
OllamaModelFactory.PROVIDER,
OllamaModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
OllamaModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, AzureModelFactory.PROVIDER,
AzureModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
DashscopeModelFactory.PROVIDER,
DashscopeModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
QianfanModelFactory.PROVIDER,
@@ -205,15 +117,13 @@ public class EmbeddingModelParameterConfig extends ParameterConfig {
}
private static List<Parameter.Dependency> getModelPathDependency() {
return getDependency(
EMBEDDING_MODEL_PROVIDER.getName(),
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(InMemoryModelFactory.PROVIDER),
ImmutableMap.of(InMemoryModelFactory.PROVIDER, ""));
}
private static List<Parameter.Dependency> getSecretKeyDependency() {
return getDependency(
EMBEDDING_MODEL_PROVIDER.getName(),
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
Lists.newArrayList(QianfanModelFactory.PROVIDER),
ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO));
}

View File

@@ -15,83 +15,38 @@ import java.util.List;
@Service("EmbeddingStoreParameterConfig")
@Slf4j
public class EmbeddingStoreParameterConfig extends ParameterConfig {
public static final Parameter EMBEDDING_STORE_PROVIDER =
new Parameter(
"s2.embedding.store.provider",
EmbeddingStoreType.IN_MEMORY.name(),
"向量库类型",
"目前支持三种类型IN_MEMORY、MILVUS、CHROMA",
"list",
"向量库配置",
getCandidateValues());
public static final Parameter EMBEDDING_STORE_PROVIDER = new Parameter(
"s2.embedding.store.provider", EmbeddingStoreType.IN_MEMORY.name(), "向量库类型",
"目前支持三种类型IN_MEMORY、MILVUS、CHROMA", "list", "向量库配置", getCandidateValues());
public static final Parameter EMBEDDING_STORE_BASE_URL =
new Parameter(
"s2.embedding.store.base.url",
"",
"BaseUrl",
"",
"string",
"向量库配置",
null,
new Parameter("s2.embedding.store.base.url", "", "BaseUrl", "", "string", "向量库配置", null,
getBaseUrlDependency());
public static final Parameter EMBEDDING_STORE_API_KEY =
new Parameter(
"s2.embedding.store.api.key",
"",
"ApiKey",
"",
"password",
"向量库配置",
null,
new Parameter("s2.embedding.store.api.key", "", "ApiKey", "", "password", "向量库配置", null,
getApiKeyDependency());
public static final Parameter EMBEDDING_STORE_PERSIST_PATH =
new Parameter(
"s2.embedding.store.persist.path",
"",
"持久化路径",
"默认不持久化,如需持久化请填写持久化路径。" + "注意:如果变更了向量模型需删除该路径下已保存的文件或修改持久化路径",
"string",
"向量库配置",
null,
getPathDependency());
new Parameter("s2.embedding.store.persist.path", "", "持久化路径",
"默认不持久化,如需持久化请填写持久化路径。" + "注意:如果变更了向量模型需删除该路径下已保存的文件或修改持久化路径", "string",
"向量库配置", null, getPathDependency());
public static final Parameter EMBEDDING_STORE_TIMEOUT =
new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", "向量库配置");
public static final Parameter EMBEDDING_STORE_DIMENSION =
new Parameter(
"s2.embedding.store.dimension",
"",
"纬度",
"",
"number",
"向量库配置",
null,
new Parameter("s2.embedding.store.dimension", "", "纬度", "", "number", "向量库配置", null,
getDimensionDependency());
public static final Parameter EMBEDDING_STORE_DATABASE_NAME =
new Parameter(
"s2.embedding.store.databaseName",
"",
"DatabaseName",
"",
"string",
"向量库配置",
null,
getDatabaseNameDependency());
new Parameter("s2.embedding.store.databaseName", "", "DatabaseName", "", "string",
"向量库配置", null, getDatabaseNameDependency());
@Override
public List<Parameter> getSysParameters() {
return Lists.newArrayList(
EMBEDDING_STORE_PROVIDER,
EMBEDDING_STORE_BASE_URL,
EMBEDDING_STORE_API_KEY,
EMBEDDING_STORE_DATABASE_NAME,
EMBEDDING_STORE_PERSIST_PATH,
EMBEDDING_STORE_TIMEOUT,
EMBEDDING_STORE_DIMENSION);
return Lists.newArrayList(EMBEDDING_STORE_PROVIDER, EMBEDDING_STORE_BASE_URL,
EMBEDDING_STORE_API_KEY, EMBEDDING_STORE_DATABASE_NAME,
EMBEDDING_STORE_PERSIST_PATH, EMBEDDING_STORE_TIMEOUT, EMBEDDING_STORE_DIMENSION);
}
public EmbeddingStoreConfig convert() {
@@ -105,58 +60,44 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
if (StringUtils.isNumeric(getParameterValue(EMBEDDING_STORE_DIMENSION))) {
dimension = Integer.valueOf(getParameterValue(EMBEDDING_STORE_DIMENSION));
}
return EmbeddingStoreConfig.builder()
.provider(provider)
.baseUrl(baseUrl)
.apiKey(apiKey)
.persistPath(persistPath)
.databaseName(databaseName)
.timeOut(Long.valueOf(timeOut))
.dimension(dimension)
.build();
return EmbeddingStoreConfig.builder().provider(provider).baseUrl(baseUrl).apiKey(apiKey)
.persistPath(persistPath).databaseName(databaseName).timeOut(Long.valueOf(timeOut))
.dimension(dimension).build();
}
private static ArrayList<String> getCandidateValues() {
return Lists.newArrayList(
EmbeddingStoreType.IN_MEMORY.name(),
EmbeddingStoreType.MILVUS.name(),
EmbeddingStoreType.CHROMA.name());
return Lists.newArrayList(EmbeddingStoreType.IN_MEMORY.name(),
EmbeddingStoreType.MILVUS.name(), EmbeddingStoreType.CHROMA.name());
}
private static List<Parameter.Dependency> getBaseUrlDependency() {
return getDependency(
EMBEDDING_STORE_PROVIDER.getName(),
Lists.newArrayList(
EmbeddingStoreType.MILVUS.name(), EmbeddingStoreType.CHROMA.name()),
ImmutableMap.of(
EmbeddingStoreType.MILVUS.name(), "http://localhost:19530",
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
EmbeddingStoreType.CHROMA.name()),
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "http://localhost:19530",
EmbeddingStoreType.CHROMA.name(), "http://localhost:8000"));
}
private static List<Parameter.Dependency> getApiKeyDependency() {
return getDependency(
EMBEDDING_STORE_PROVIDER.getName(),
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO));
}
private static List<Parameter.Dependency> getPathDependency() {
return getDependency(
EMBEDDING_STORE_PROVIDER.getName(),
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
Lists.newArrayList(EmbeddingStoreType.IN_MEMORY.name()),
ImmutableMap.of(EmbeddingStoreType.IN_MEMORY.name(), ""));
}
private static List<Parameter.Dependency> getDimensionDependency() {
return getDependency(
EMBEDDING_STORE_PROVIDER.getName(),
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "384"));
}
private static List<Parameter.Dependency> getDatabaseNameDependency() {
return getDependency(
EMBEDDING_STORE_PROVIDER.getName(),
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), ""));
}

View File

@@ -15,9 +15,11 @@ import java.util.Map;
@Service
public abstract class ParameterConfig {
public static final String DEMO = "demo";
@Autowired private SystemConfigService sysConfigService;
@Autowired
private SystemConfigService sysConfigService;
@Autowired private Environment environment;
@Autowired
private Environment environment;
/** @return system parameters to be set with user interface */
protected List<Parameter> getSysParameters() {
@@ -46,10 +48,8 @@ public abstract class ParameterConfig {
return value;
}
protected static List<Parameter.Dependency> getDependency(
String dependencyParameterName,
List<String> includesValue,
Map<String, String> setDefaultValue) {
protected static List<Parameter.Dependency> getDependency(String dependencyParameterName,
List<String> includesValue, Map<String, String> setDefaultValue) {
Parameter.Dependency.Show show = new Parameter.Dependency.Show();
show.setIncludesValue(includesValue);

View File

@@ -38,11 +38,8 @@ public class SystemConfig {
if (StringUtils.isBlank(name)) {
return "";
}
Map<String, String> nameToValue =
getParameters().stream()
.collect(
Collectors.toMap(
Parameter::getName, Parameter::getValue, (k1, k2) -> k1));
Map<String, String> nameToValue = getParameters().stream()
.collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (k1, k2) -> k1));
return nameToValue.get(name);
}
@@ -69,15 +66,11 @@ public class SystemConfig {
if (CollectionUtils.isEmpty(parameters)) {
return defaultParameters;
}
Map<String, String> parameterNameValueMap =
parameters.stream()
.collect(
Collectors.toMap(
Parameter::getName, Parameter::getValue, (v1, v2) -> v2));
Map<String, String> parameterNameValueMap = parameters.stream()
.collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (v1, v2) -> v2));
for (Parameter parameter : defaultParameters) {
parameter.setValue(
parameterNameValueMap.getOrDefault(
parameter.getName(), parameter.getDefaultValue()));
parameter.setValue(parameterNameValueMap.getOrDefault(parameter.getName(),
parameter.getDefaultValue()));
}
return defaultParameters;
}

View File

@@ -14,8 +14,8 @@ import org.springframework.web.servlet.ModelAndView;
@Slf4j
public class LogInterceptor implements HandlerInterceptor {
@Override
public boolean preHandle(
HttpServletRequest request, HttpServletResponse response, Object handler) {
public boolean preHandle(HttpServletRequest request, HttpServletResponse response,
Object handler) {
// use previous traceId
String traceId = request.getHeader(TraceIdUtil.TRACE_ID);
if (StringUtils.isBlank(traceId)) {
@@ -27,17 +27,12 @@ public class LogInterceptor implements HandlerInterceptor {
}
@Override
public void postHandle(
HttpServletRequest request,
HttpServletResponse response,
Object handler,
ModelAndView modelAndView)
throws Exception {}
public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler,
ModelAndView modelAndView) throws Exception {}
@Override
public void afterCompletion(
HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex)
throws Exception {
public void afterCompletion(HttpServletRequest request, HttpServletResponse response,
Object handler, Exception ex) throws Exception {
// remove after Completing
TraceIdUtil.remove();
}

View File

@@ -5,13 +5,9 @@ import java.util.Map;
import java.util.stream.Collectors;
public enum AggregateEnum {
MOST("最多", "max"),
HIGHEST("", "max"),
MAXIMUN("最大", "max"),
LEAST("最少", "min"),
SMALLEST("最小", "min"),
LOWEST("最低", "min"),
AVERAGE("平均", "avg");
MOST("最多", "max"), HIGHEST("最高", "max"), MAXIMUN("最大", "max"), LEAST("最少",
"min"), SMALLEST("", "min"), LOWEST("最低", "min"), AVERAGE("平均", "avg");
private String aggregateCh;
private String aggregateEN;
@@ -29,9 +25,7 @@ public enum AggregateEnum {
}
public static Map<String, String> getAggregateEnum() {
return Arrays.stream(AggregateEnum.values())
.collect(
Collectors.toMap(
AggregateEnum::getAggregateCh, AggregateEnum::getAggregateEN));
return Arrays.stream(AggregateEnum.values()).collect(
Collectors.toMap(AggregateEnum::getAggregateCh, AggregateEnum::getAggregateEN));
}
}

View File

@@ -15,8 +15,8 @@ public class CustomExpressionDeParser extends ExpressionDeParser {
private boolean dealNull;
private boolean dealNotNull;
public CustomExpressionDeParser(
Set<String> removeFieldNames, boolean dealNull, boolean dealNotNull) {
public CustomExpressionDeParser(Set<String> removeFieldNames, boolean dealNull,
boolean dealNotNull) {
this.removeFieldNames = removeFieldNames;
this.dealNull = dealNull;
this.dealNotNull = dealNotNull;
@@ -45,11 +45,9 @@ public class CustomExpressionDeParser extends ExpressionDeParser {
Expression leftExpression = ((AndExpression) binaryExpression).getLeftExpression();
Expression rightExpression = ((AndExpression) binaryExpression).getRightExpression();
boolean leftIsNull =
leftExpression instanceof IsNullExpression
boolean leftIsNull = leftExpression instanceof IsNullExpression
&& shouldSkip((IsNullExpression) leftExpression);
boolean rightIsNull =
rightExpression instanceof IsNullExpression
boolean rightIsNull = rightExpression instanceof IsNullExpression
&& shouldSkip((IsNullExpression) rightExpression);
if (leftIsNull && rightIsNull) {

View File

@@ -13,8 +13,8 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
@Slf4j
public class DateFunctionHelper {
public static String getStartDateStr(
ComparisonOperator minorThanEquals, ExpressionList<?> expressions) {
public static String getStartDateStr(ComparisonOperator minorThanEquals,
ExpressionList<?> expressions) {
String unitValue = getUnit(expressions);
String dateValue = getEndDateValue(expressions);
String dateStr = "";

View File

@@ -23,8 +23,7 @@ public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter {
expr.getWhenExpression().accept(this);
if (expr.getThenExpression() instanceof Column) {
Column column = (Column) expr.getThenExpression();
Expression expression =
QueryExpressionReplaceVisitor.getExpression(
Expression expression = QueryExpressionReplaceVisitor.getExpression(
QueryExpressionReplaceVisitor.getReplaceExpr(column, fieldExprMap));
if (Objects.nonNull(expression)) {
expr.setThenExpression(expression);
@@ -52,20 +51,16 @@ public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter {
}
}
if (left instanceof Column) {
Expression expression =
QueryExpressionReplaceVisitor.getExpression(
QueryExpressionReplaceVisitor.getReplaceExpr(
(Column) left, fieldExprMap));
Expression expression = QueryExpressionReplaceVisitor.getExpression(
QueryExpressionReplaceVisitor.getReplaceExpr((Column) left, fieldExprMap));
if (Objects.nonNull(expression)) {
expr.setLeftExpression(expression);
leftVisited = true;
}
}
if (right instanceof Column) {
Expression expression =
QueryExpressionReplaceVisitor.getExpression(
QueryExpressionReplaceVisitor.getReplaceExpr(
(Column) right, fieldExprMap));
Expression expression = QueryExpressionReplaceVisitor.getExpression(
QueryExpressionReplaceVisitor.getReplaceExpr((Column) right, fieldExprMap));
if (Objects.nonNull(expression)) {
expr.setRightExpression(expression);
rightVisited = true;
@@ -81,8 +76,7 @@ public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter {
private boolean visitFunction(Function function) {
if (function.getParameters().getExpressions().get(0) instanceof Column) {
Expression expression =
QueryExpressionReplaceVisitor.getExpression(
Expression expression = QueryExpressionReplaceVisitor.getExpression(
QueryExpressionReplaceVisitor.getReplaceExpr(function, fieldExprMap));
if (Objects.nonNull(expression)) {
ExpressionList<Expression> expressions = new ExpressionList<>();

View File

@@ -130,8 +130,8 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
Arrays.stream(DatePeriodEnum.values()).collect(Collectors.toList());
DatePeriodEnum periodEnum = DatePeriodEnum.get(functionName);
if (Objects.nonNull(periodEnum) && collect.contains(periodEnum)) {
fieldExpression.setFieldValue(
getFieldValue(rightExpression) + periodEnum.getChName());
fieldExpression
.setFieldValue(getFieldValue(rightExpression) + periodEnum.getChName());
return fieldExpression;
} else {
// deal with aggregate function

View File

@@ -31,8 +31,8 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter {
private boolean exactReplace;
private Map<String, Map<String, String>> filedNameToValueMap;
public FieldValueReplaceVisitor(
boolean exactReplace, Map<String, Map<String, String>> filedNameToValueMap) {
public FieldValueReplaceVisitor(boolean exactReplace,
Map<String, Map<String, String>> filedNameToValueMap) {
this.exactReplace = exactReplace;
this.filedNameToValueMap = filedNameToValueMap;
}
@@ -67,9 +67,7 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter {
ExpressionList rightItemsList = (ExpressionList) inExpression.getRightExpression();
List<Expression> expressions = rightItemsList.getExpressions();
List<String> values = new ArrayList<>();
expressions.stream()
.forEach(
o -> {
expressions.stream().forEach(o -> {
if (o instanceof StringValue) {
values.add(((StringValue) o).getValue());
}
@@ -78,9 +76,7 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter {
return;
}
List<Expression> newExpressions = new ArrayList<>();
values.stream()
.forEach(
o -> {
values.stream().forEach(o -> {
String replaceValue = valueMap.getOrDefault(o, o);
StringValue stringValue = new StringValue(replaceValue);
newExpressions.add(stringValue);

View File

@@ -34,11 +34,9 @@ public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
if (!(rightExpression instanceof StringValue)
|| !(leftExpression instanceof Column)
if (!(rightExpression instanceof StringValue) || !(leftExpression instanceof Column)
|| CollectionUtils.isEmpty(fieldValueToFieldNames)
|| Objects.isNull(rightExpression)
|| Objects.isNull(leftExpression)) {
|| Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) {
return;
}

View File

@@ -21,8 +21,8 @@ public class FunctionAliasReplaceVisitor extends SelectItemVisitorAdapter {
// 2.alias's fieldName not equal. "sum(pv) as pv" cannot be replaced.
if (Objects.nonNull(selectExpressionItem.getAlias())
&& !selectExpressionItem.getAlias().getName().equalsIgnoreCase(columnName)) {
aliasToActualExpression.put(
selectExpressionItem.getAlias().getName(), function.toString());
aliasToActualExpression.put(selectExpressionItem.getAlias().getName(),
function.toString());
selectExpressionItem.setAlias(null);
}
}

View File

@@ -16,8 +16,8 @@ public class FunctionNameReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, String> functionMap;
private Map<String, UnaryOperator> functionCallMap;
public FunctionNameReplaceVisitor(
Map<String, String> functionMap, Map<String, UnaryOperator> functionCallMap) {
public FunctionNameReplaceVisitor(Map<String, String> functionMap,
Map<String, UnaryOperator> functionCallMap) {
this.functionMap = functionMap;
this.functionCallMap = functionCallMap;
}

View File

@@ -19,8 +19,8 @@ public class GroupByFunctionReplaceVisitor implements GroupByVisitor {
private Map<String, String> functionMap;
private Map<String, UnaryOperator> functionCallMap;
public GroupByFunctionReplaceVisitor(
Map<String, String> functionMap, Map<String, UnaryOperator> functionCallMap) {
public GroupByFunctionReplaceVisitor(Map<String, String> functionMap,
Map<String, UnaryOperator> functionCallMap) {
this.functionMap = functionMap;
this.functionCallMap = functionCallMap;
}

View File

@@ -53,11 +53,8 @@ public class GroupByReplaceVisitor implements GroupByVisitor {
return expression.toString();
}
private void replaceExpression(
List<Expression> groupByExpressions,
int index,
Expression expression,
String replaceColumn) {
private void replaceExpression(List<Expression> groupByExpressions, int index,
Expression expression, String replaceColumn) {
if (expression instanceof Column) {
groupByExpressions.set(index, new Column(replaceColumn));
} else if (expression instanceof Function) {
@@ -68,8 +65,7 @@ public class GroupByReplaceVisitor implements GroupByVisitor {
Function function = (Function) expression;
if (function.getParameters().size() > 1) {
function.getParameters().stream()
.skip(1)
function.getParameters().stream().skip(1)
.forEach(e -> newExpressionList.add((Function) e));
}
function.setParameters(newExpressionList);

View File

@@ -27,26 +27,14 @@ public class JsqlConstants {
public static final String IN_CONSTANT = " 1 in (1) ";
public static final String LIKE_CONSTANT = "1 like 1";
public static final String IN = "IN";
public static final Map<String, String> rightMap =
Stream.of(
new AbstractMap.SimpleEntry<>("<=", "<="),
new AbstractMap.SimpleEntry<>("<", "<"),
new AbstractMap.SimpleEntry<>(">=", "<="),
new AbstractMap.SimpleEntry<>(">", "<"),
public static final Map<String, String> rightMap = Stream.of(
new AbstractMap.SimpleEntry<>("<=", "<="), new AbstractMap.SimpleEntry<>("<", "<"),
new AbstractMap.SimpleEntry<>(">=", "<="), new AbstractMap.SimpleEntry<>(">", "<"),
new AbstractMap.SimpleEntry<>("=", "<="))
.collect(
toMap(
AbstractMap.SimpleEntry::getKey,
AbstractMap.SimpleEntry::getValue));
public static final Map<String, String> leftMap =
Stream.of(
new AbstractMap.SimpleEntry<>("<=", ">="),
new AbstractMap.SimpleEntry<>("<", ">"),
new AbstractMap.SimpleEntry<>(">=", "<="),
new AbstractMap.SimpleEntry<>(">", "<"),
.collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
public static final Map<String, String> leftMap = Stream.of(
new AbstractMap.SimpleEntry<>("<=", ">="), new AbstractMap.SimpleEntry<>("<", ">"),
new AbstractMap.SimpleEntry<>(">=", "<="), new AbstractMap.SimpleEntry<>(">", "<"),
new AbstractMap.SimpleEntry<>("=", ">="))
.collect(
toMap(
AbstractMap.SimpleEntry::getKey,
AbstractMap.SimpleEntry::getValue));
.collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
}

View File

@@ -13,8 +13,8 @@ import java.util.stream.Collectors;
@Slf4j
public class ParseVisitorHelper {
public void replaceColumn(
Column column, Map<String, String> fieldNameMap, boolean exactReplace) {
public void replaceColumn(Column column, Map<String, String> fieldNameMap,
boolean exactReplace) {
String columnName = StringUtil.replaceBackticks(column.getColumnName());
String replaceColumn = getReplaceValue(columnName, fieldNameMap, exactReplace);
if (StringUtils.isNotBlank(replaceColumn)) {
@@ -22,8 +22,8 @@ public class ParseVisitorHelper {
}
}
public String getReplaceValue(
String beforeValue, Map<String, String> valueMap, boolean exactReplace) {
public String getReplaceValue(String beforeValue, Map<String, String> valueMap,
boolean exactReplace) {
String value = valueMap.get(beforeValue);
if (StringUtils.isNotBlank(value)) {
return value;
@@ -31,19 +31,13 @@ public class ParseVisitorHelper {
if (exactReplace) {
return null;
}
Optional<Entry<String, String>> first =
valueMap.entrySet().stream()
.sorted(
(k1, k2) -> {
Optional<Entry<String, String>> first = valueMap.entrySet().stream().sorted((k1, k2) -> {
String k1Value = k1.getKey();
String k2Value = k2.getKey();
Double k1Similarity = getSimilarity(beforeValue, k1Value);
Double k2Similarity = getSimilarity(beforeValue, k2Value);
return k2Similarity.compareTo(k1Similarity);
})
.collect(Collectors.toList())
.stream()
.findFirst();
}).collect(Collectors.toList()).stream().findFirst();
if (first.isPresent()) {
return first.get().getValue();
@@ -68,15 +62,11 @@ public class ParseVisitorHelper {
char cj = word2.charAt(j - 1);
if (ci == cj) {
dp[i][j] = dp[i - 1][j - 1];
} else if (i > 1
&& j > 1
&& ci == word2.charAt(j - 2)
} else if (i > 1 && j > 1 && ci == word2.charAt(j - 2)
&& cj == word1.charAt(i - 2)) {
dp[i][j] = 1 + Math.min(dp[i - 2][j - 2], Math.min(dp[i][j - 1], dp[i - 1][j]));
} else {
dp[i][j] =
Math.min(
dp[i - 1][j - 1] + 1,
dp[i][j] = Math.min(dp[i - 1][j - 1] + 1,
Math.min(dp[i][j - 1] + 1, dp[i - 1][j] + 1));
}
}

View File

@@ -43,30 +43,19 @@ public class SqlAddHelper {
}
if (selectStatement instanceof PlainSelect) {
PlainSelect plainSelect = (PlainSelect) selectStatement;
fields.stream()
.filter(Objects::nonNull)
.forEach(
field -> {
SelectItem<Column> selectExpressionItem =
new SelectItem(new Column(field));
fields.stream().filter(Objects::nonNull).forEach(field -> {
SelectItem<Column> selectExpressionItem = new SelectItem(new Column(field));
plainSelect.addSelectItems(selectExpressionItem);
});
} else if (selectStatement instanceof SetOperationList) {
SetOperationList setOperationList = (SetOperationList) selectStatement;
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
setOperationList
.getSelects()
.forEach(
subSelectBody -> {
setOperationList.getSelects().forEach(subSelectBody -> {
PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
fields.stream()
.forEach(
field -> {
SelectItem<Column> selectExpressionItem =
new SelectItem(new Column(field));
subPlainSelect.addSelectItems(
selectExpressionItem);
fields.stream().forEach(field -> {
SelectItem<Column> selectExpressionItem = new SelectItem(new Column(field));
subPlainSelect.addSelectItems(selectExpressionItem);
});
});
}
@@ -88,10 +77,7 @@ public class SqlAddHelper {
SetOperationList setOperationList =
(SetOperationList) selectStatement.getSetOperationList();
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
setOperationList
.getSelects()
.forEach(
subSelectBody -> {
setOperationList.getSelects().forEach(subSelectBody -> {
PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
plainSelectList.add(subPlainSelect);
});
@@ -238,14 +224,11 @@ public class SqlAddHelper {
if (!(selectStatement instanceof PlainSelect)) {
return sql;
}
selectStatement.accept(
new SelectVisitorAdapter() {
selectStatement.accept(new SelectVisitorAdapter() {
@Override
public void visit(PlainSelect plainSelect) {
addAggregateToSelectItems(
plainSelect.getSelectItems(), fieldNameToAggregate);
addAggregateToOrderByItems(
plainSelect.getOrderByElements(), fieldNameToAggregate);
addAggregateToSelectItems(plainSelect.getSelectItems(), fieldNameToAggregate);
addAggregateToOrderByItems(plainSelect.getOrderByElements(), fieldNameToAggregate);
addAggregateToGroupByItems(plainSelect.getGroupBy(), fieldNameToAggregate);
addAggregateToWhereItems(plainSelect.getWhere(), fieldNameToAggregate);
}
@@ -276,8 +259,8 @@ public class SqlAddHelper {
return selectStatement.toString();
}
private static void addAggregateToSelectItems(
List<SelectItem<?>> selectItems, Map<String, String> fieldNameToAggregate) {
private static void addAggregateToSelectItems(List<SelectItem<?>> selectItems,
Map<String, String> fieldNameToAggregate) {
for (SelectItem selectItem : selectItems) {
Expression expression = selectItem.getExpression();
Function function =
@@ -289,8 +272,8 @@ public class SqlAddHelper {
}
}
private static void addAggregateToOrderByItems(
List<OrderByElement> orderByElements, Map<String, String> fieldNameToAggregate) {
private static void addAggregateToOrderByItems(List<OrderByElement> orderByElements,
Map<String, String> fieldNameToAggregate) {
if (orderByElements == null) {
return;
}
@@ -305,8 +288,8 @@ public class SqlAddHelper {
}
}
private static void addAggregateToGroupByItems(
GroupByElement groupByElement, Map<String, String> fieldNameToAggregate) {
private static void addAggregateToGroupByItems(GroupByElement groupByElement,
Map<String, String> fieldNameToAggregate) {
if (groupByElement == null) {
return;
}
@@ -321,16 +304,16 @@ public class SqlAddHelper {
}
}
private static void addAggregateToWhereItems(
Expression whereExpression, Map<String, String> fieldNameToAggregate) {
private static void addAggregateToWhereItems(Expression whereExpression,
Map<String, String> fieldNameToAggregate) {
if (whereExpression == null) {
return;
}
modifyWhereExpression(whereExpression, fieldNameToAggregate);
}
private static void modifyWhereExpression(
Expression whereExpression, Map<String, String> fieldNameToAggregate) {
private static void modifyWhereExpression(Expression whereExpression,
Map<String, String> fieldNameToAggregate) {
if (SqlSelectHelper.isLogicExpression(whereExpression)) {
if (whereExpression instanceof AndExpression) {
AndExpression andExpression = (AndExpression) whereExpression;
@@ -347,15 +330,15 @@ public class SqlAddHelper {
modifyWhereExpression(rightExpression, fieldNameToAggregate);
}
} else if (whereExpression instanceof Parenthesis) {
modifyWhereExpression(
((Parenthesis) whereExpression).getExpression(), fieldNameToAggregate);
modifyWhereExpression(((Parenthesis) whereExpression).getExpression(),
fieldNameToAggregate);
} else {
setAggToFunction(whereExpression, fieldNameToAggregate);
}
}
private static void setAggToFunction(
Expression expression, Map<String, String> fieldNameToAggregate) {
private static void setAggToFunction(Expression expression,
Map<String, String> fieldNameToAggregate) {
if (!(expression instanceof ComparisonOperator)) {
return;
}
@@ -363,20 +346,16 @@ public class SqlAddHelper {
if (comparisonOperator.getRightExpression() instanceof Column) {
String columnName =
((Column) (comparisonOperator).getRightExpression()).getColumnName();
Function function =
SqlSelectFunctionHelper.getFunction(
comparisonOperator.getRightExpression(),
fieldNameToAggregate.get(columnName));
Function function = SqlSelectFunctionHelper.getFunction(
comparisonOperator.getRightExpression(), fieldNameToAggregate.get(columnName));
if (Objects.nonNull(function)) {
comparisonOperator.setRightExpression(function);
}
}
if (comparisonOperator.getLeftExpression() instanceof Column) {
String columnName = ((Column) (comparisonOperator).getLeftExpression()).getColumnName();
Function function =
SqlSelectFunctionHelper.getFunction(
comparisonOperator.getLeftExpression(),
fieldNameToAggregate.get(columnName));
Function function = SqlSelectFunctionHelper.getFunction(
comparisonOperator.getLeftExpression(), fieldNameToAggregate.get(columnName));
if (Objects.nonNull(function)) {
comparisonOperator.setLeftExpression(function);
}

View File

@@ -27,8 +27,7 @@ public class SqlAsHelper {
if (plainSelect instanceof Select) {
Select select = plainSelect;
Select selectBody = select.getSelectBody();
selectBody.accept(
new SelectVisitorAdapter() {
selectBody.accept(new SelectVisitorAdapter() {
@Override
public void visit(PlainSelect plainSelect) {
extractAliasesFromSelect(plainSelect, aliases);

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.common.jsqlparser;
public enum SqlEditEnum {
NUMBER_FILTER,
DATEDIFF
NUMBER_FILTER, DATEDIFF
}

View File

@@ -67,8 +67,7 @@ public class SqlRemoveHelper {
}
List<SelectItem<?>> selectItems = ((PlainSelect) selectStatement).getSelectItems();
Set<String> fields = new HashSet<>();
selectItems.removeIf(
selectItem -> {
selectItems.removeIf(selectItem -> {
String field = selectItem.getExpression().toString();
if (fields.contains(field)) {
return true;
@@ -85,8 +84,7 @@ public class SqlRemoveHelper {
if (!(selectStatement instanceof PlainSelect)) {
return sql;
}
selectStatement.accept(
new SelectVisitorAdapter() {
selectStatement.accept(new SelectVisitorAdapter() {
@Override
public void visit(PlainSelect plainSelect) {
removeWhereCondition(plainSelect.getWhere(), removeFieldNames);
@@ -95,8 +93,8 @@ public class SqlRemoveHelper {
return removeNumberFilter(selectStatement.toString());
}
private static void removeWhereCondition(
Expression whereExpression, Set<String> removeFieldNames) {
private static void removeWhereCondition(Expression whereExpression,
Set<String> removeFieldNames) {
if (whereExpression == null) {
return;
}
@@ -121,8 +119,8 @@ public class SqlRemoveHelper {
return selectStatement.toString();
}
private static void removeWhereExpression(
Expression whereExpression, Set<String> removeFieldNames) {
private static void removeWhereExpression(Expression whereExpression,
Set<String> removeFieldNames) {
if (SqlSelectHelper.isLogicExpression(whereExpression)) {
BinaryExpression binaryExpression = (BinaryExpression) whereExpression;
Expression leftExpression = binaryExpression.getLeftExpression();
@@ -131,8 +129,8 @@ public class SqlRemoveHelper {
removeWhereExpression(leftExpression, removeFieldNames);
removeWhereExpression(rightExpression, removeFieldNames);
} else if (whereExpression instanceof Parenthesis) {
removeWhereExpression(
((Parenthesis) whereExpression).getExpression(), removeFieldNames);
removeWhereExpression(((Parenthesis) whereExpression).getExpression(),
removeFieldNames);
} else {
removeExpressionWithConstant(whereExpression, removeFieldNames);
}
@@ -152,8 +150,8 @@ public class SqlRemoveHelper {
return constant;
}
private static void removeExpressionWithConstant(
Expression expression, Set<String> removeFieldNames) {
private static void removeExpressionWithConstant(Expression expression,
Set<String> removeFieldNames) {
try {
if (expression instanceof ComparisonOperator) {
handleComparisonOperator((ComparisonOperator) expression, removeFieldNames);
@@ -167,12 +165,9 @@ public class SqlRemoveHelper {
}
}
private static void handleComparisonOperator(
ComparisonOperator comparisonOperator, Set<String> removeFieldNames)
throws JSQLParserException {
String columnName =
SqlSelectHelper.getColumnName(
comparisonOperator.getLeftExpression(),
private static void handleComparisonOperator(ComparisonOperator comparisonOperator,
Set<String> removeFieldNames) throws JSQLParserException {
String columnName = SqlSelectHelper.getColumnName(comparisonOperator.getLeftExpression(),
comparisonOperator.getRightExpression());
if (!removeFieldNames.contains(columnName)) {
return;
@@ -185,9 +180,8 @@ public class SqlRemoveHelper {
private static void handleInExpression(InExpression inExpression, Set<String> removeFieldNames)
throws JSQLParserException {
String columnName =
SqlSelectHelper.getColumnName(
inExpression.getLeftExpression(), inExpression.getRightExpression());
String columnName = SqlSelectHelper.getColumnName(inExpression.getLeftExpression(),
inExpression.getRightExpression());
if (!removeFieldNames.contains(columnName)) {
return;
}
@@ -196,12 +190,10 @@ public class SqlRemoveHelper {
updateInExpression(inExpression, constantExpression);
}
private static void handleLikeExpression(
LikeExpression likeExpression, Set<String> removeFieldNames)
throws JSQLParserException {
String columnName =
SqlSelectHelper.getColumnName(
likeExpression.getLeftExpression(), likeExpression.getRightExpression());
private static void handleLikeExpression(LikeExpression likeExpression,
Set<String> removeFieldNames) throws JSQLParserException {
String columnName = SqlSelectHelper.getColumnName(likeExpression.getLeftExpression(),
likeExpression.getRightExpression());
if (!removeFieldNames.contains(columnName)) {
return;
}
@@ -210,8 +202,8 @@ public class SqlRemoveHelper {
updateLikeExpression(likeExpression, constantExpression);
}
private static void updateComparisonOperator(
ComparisonOperator original, ComparisonOperator constantExpression) {
private static void updateComparisonOperator(ComparisonOperator original,
ComparisonOperator constantExpression) {
original.setLeftExpression(constantExpression.getLeftExpression());
original.setRightExpression(constantExpression.getRightExpression());
original.setASTNode(constantExpression.getASTNode());
@@ -223,8 +215,8 @@ public class SqlRemoveHelper {
original.setASTNode(constantExpression.getASTNode());
}
private static void updateLikeExpression(
LikeExpression original, LikeExpression constantExpression) {
private static void updateLikeExpression(LikeExpression original,
LikeExpression constantExpression) {
original.setLeftExpression(constantExpression.getLeftExpression());
original.setRightExpression(constantExpression.getRightExpression());
}
@@ -234,8 +226,7 @@ public class SqlRemoveHelper {
if (!(selectStatement instanceof PlainSelect)) {
return sql;
}
selectStatement.accept(
new SelectVisitorAdapter() {
selectStatement.accept(new SelectVisitorAdapter() {
@Override
public void visit(PlainSelect plainSelect) {
removeWhereCondition(plainSelect.getHaving(), removeFieldNames);
@@ -254,10 +245,7 @@ public class SqlRemoveHelper {
return sql;
}
ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList();
groupByExpressionList
.getExpressions()
.removeIf(
expression -> {
groupByExpressionList.getExpressions().removeIf(expression -> {
if (expression instanceof Column) {
Column column = (Column) expression;
return fields.contains(column.getColumnName());
@@ -279,8 +267,7 @@ public class SqlRemoveHelper {
Iterator<SelectItem<?>> iterator = selectItems.iterator();
while (iterator.hasNext()) {
SelectItem selectItem = iterator.next();
selectItem.accept(
new SelectItemVisitorAdapter() {
selectItem.accept(new SelectItemVisitorAdapter() {
@Override
public void visit(SelectItem item) {
if (fields.contains(item.getExpression().toString())) {
@@ -345,17 +332,14 @@ public class SqlRemoveHelper {
}
}
private static Expression dealComparisonOperatorFilter(
Expression expression, SqlEditEnum sqlEditEnum) {
private static Expression dealComparisonOperatorFilter(Expression expression,
SqlEditEnum sqlEditEnum) {
if (Objects.isNull(expression)) {
return null;
}
if (expression instanceof GreaterThanEquals
|| expression instanceof GreaterThan
|| expression instanceof MinorThan
|| expression instanceof MinorThanEquals
|| expression instanceof EqualsTo
|| expression instanceof NotEqualsTo) {
if (expression instanceof GreaterThanEquals || expression instanceof GreaterThan
|| expression instanceof MinorThan || expression instanceof MinorThanEquals
|| expression instanceof EqualsTo || expression instanceof NotEqualsTo) {
return removeSingleFilter((ComparisonOperator) expression, sqlEditEnum);
} else if (expression instanceof InExpression) {
InExpression inExpression = (InExpression) expression;
@@ -369,14 +353,14 @@ public class SqlRemoveHelper {
return expression;
}
private static Expression removeSingleFilter(
ComparisonOperator comparisonExpression, SqlEditEnum sqlEditEnum) {
private static Expression removeSingleFilter(ComparisonOperator comparisonExpression,
SqlEditEnum sqlEditEnum) {
Expression leftExpression = comparisonExpression.getLeftExpression();
return recursionBase(leftExpression, comparisonExpression, sqlEditEnum);
}
private static Expression recursionBase(
Expression leftExpression, Expression expression, SqlEditEnum sqlEditEnum) {
private static Expression recursionBase(Expression leftExpression, Expression expression,
SqlEditEnum sqlEditEnum) {
if (sqlEditEnum.equals(SqlEditEnum.NUMBER_FILTER)) {
return distinguishNumberFilter(leftExpression, expression);
}
@@ -386,8 +370,8 @@ public class SqlRemoveHelper {
return expression;
}
private static Expression distinguishNumberFilter(
Expression leftExpression, Expression expression) {
private static Expression distinguishNumberFilter(Expression leftExpression,
Expression expression) {
if (leftExpression instanceof LongValue) {
return null;
} else {
@@ -403,8 +387,8 @@ public class SqlRemoveHelper {
return removeIsNullOrNotNullInWhere(false, true, sql, removeFieldNames);
}
public static String removeIsNullOrNotNullInWhere(
boolean dealNull, boolean dealNotNull, String sql, Set<String> removeFieldNames) {
public static String removeIsNullOrNotNullInWhere(boolean dealNull, boolean dealNotNull,
String sql, Set<String> removeFieldNames) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
if (!(selectStatement instanceof PlainSelect)) {
return sql;

Some files were not shown because too many files have changed in this diff Show More