1 Commits

Author SHA1 Message Date
FredTsang
a32014d40b Merge f489ea2f59 into 0721df2e66 2025-06-15 23:24:16 +08:00
25 changed files with 83 additions and 374 deletions

View File

@@ -43,26 +43,10 @@ if "%service%"=="webapp" (
call mvn -f %projectDir% clean package -DskipTests -Dspotless.skip=true
IF ERRORLEVEL 1 (
ECHO Failed to build backend Java modules.
ECHO Please check Maven and Java versions are compatible.
ECHO Current Java: %JAVA_HOME%
ECHO Current Maven: %MAVEN_HOME%
EXIT /B 1
)
REM extract and copy files to deployment directory
cd %projectDir%\launchers\%model_name%\target
if exist "launchers-%model_name%-%MVN_VERSION%-bin.tar.gz" (
echo "Extracting launchers-%model_name%-%MVN_VERSION%-bin.tar.gz..."
tar -xf "launchers-%model_name%-%MVN_VERSION%-bin.tar.gz"
if exist "launchers-%model_name%-%MVN_VERSION%" (
echo "Copying files to deployment directory..."
xcopy /E /Y "launchers-%model_name%-%MVN_VERSION%\*" "%buildDir%\supersonic-%model_name%-%MVN_VERSION%\"
)
)
copy /y %projectDir%\launchers\%model_name%\target\*.tar.gz %buildDir%\
echo "finished building supersonic-%model_name% service"
cd %baseDir%
goto :EOF
@@ -88,55 +72,22 @@ if "%service%"=="webapp" (
cd %buildDir%
if exist %release_dir% rmdir /s /q %release_dir%
if exist %release_dir%.zip del %release_dir%.zip
rem check if release directory already exists from buildJavaService
if exist %release_dir% (
echo "Release directory already prepared by buildJavaService"
) else (
mkdir %release_dir%
rem package java service
tar xvf %service_name%-bin.tar.gz 2>nul
if errorlevel 1 (
echo "Warning: tar command failed, trying PowerShell extraction..."
powershell -Command "Expand-Archive -Path '%service_name%-bin.tar.gz' -DestinationPath '.' -Force"
)
for /d %%D in ("%service_name%\*") do (
move "%%D" "%release_dir%"
)
rmdir /s /q %service_name% 2>nul
)
mkdir %release_dir%
rem package webapp
if exist supersonic-webapp.tar.gz (
tar xvf supersonic-webapp.tar.gz 2>nul
if errorlevel 1 (
echo "Warning: tar command failed, trying PowerShell extraction..."
powershell -Command "Expand-Archive -Path 'supersonic-webapp.tar.gz' -DestinationPath '.' -Force"
)
move /y supersonic-webapp webapp
echo {"env": ""} > webapp\supersonic.config.json
move /y webapp %release_dir%
del supersonic-webapp.tar.gz 2>nul
tar xvf supersonic-webapp.tar.gz
move /y supersonic-webapp webapp
echo {"env": ""} > webapp\supersonic.config.json
move /y webapp %release_dir%
rem package java service
tar xvf %service_name%-bin.tar.gz
for /d %%D in ("%service_name%\*") do (
move "%%D" "%release_dir%"
)
rem verify deployment structure
if exist "%release_dir%\lib\launchers-%model_name%-%MVN_VERSION%.jar" (
echo "Deployment structure verified successfully"
) else (
echo "Warning: Main jar file not found in deployment structure"
echo "Expected: %release_dir%\lib\launchers-%model_name%-%MVN_VERSION%.jar"
)
rem generate zip file
powershell -Command "Compress-Archive -Path '%release_dir%' -DestinationPath '%release_dir%.zip' -Force"
if errorlevel 1 (
echo "Warning: PowerShell compression failed, release directory still available: %release_dir%"
) else (
echo "Successfully created release package: %release_dir%.zip"
)
del %service_name%-bin.tar.gz 2>nul
powershell Compress-Archive -Path %release_dir% -DestinationPath %release_dir%.zip
del %service_name%-bin.tar.gz
del supersonic-webapp.tar.gz
rmdir /s /q %service_name%
echo "finished packaging supersonic release"
goto :EOF

View File

@@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import java.sql.Timestamp;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -223,9 +222,8 @@ public class DefaultUserAdaptor implements UserAdaptor {
new UserWithPassword(userDO.getId(), userDO.getName(), userDO.getDisplayName(),
userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin());
// 使用令牌名称作为生成key 这样可以区分正常请求和api 请求api 的令牌失效时间很长,需考虑令牌泄露的情况
String token = tokenService.generateToken(UserWithPassword.convert(userWithPassword),
"SysDbToken:" + name, (new Date().getTime() + expireTime));
String token =
tokenService.generateToken(UserWithPassword.convert(userWithPassword), expireTime);
UserTokenDO userTokenDO = saveUserToken(name, userName, token, expireTime);
return convertUserToken(userTokenDO);
}

View File

@@ -21,8 +21,6 @@ public interface UserRepository {
UserTokenDO getUserToken(Long tokenId);
UserTokenDO getUserTokenByName(String tokenName);
void deleteUserTokenByName(String userName);
void deleteUserToken(Long tokenId);

View File

@@ -65,13 +65,6 @@ public class UserRepositoryImpl implements UserRepository {
return userTokenDOMapper.selectById(tokenId);
}
@Override
public UserTokenDO getUserTokenByName(String tokenName) {
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(UserTokenDO::getName, tokenName);
return userTokenDOMapper.selectOne(queryWrapper);
}
@Override
public void deleteUserTokenByName(String userName) {
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();

View File

@@ -6,10 +6,7 @@ import javax.crypto.spec.SecretKeySpec;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserTokenDO;
import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository;
import com.tencent.supersonic.common.pojo.exception.AccessException;
import com.tencent.supersonic.common.util.ContextUtils;
import io.jsonwebtoken.Claims;
import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.SignatureAlgorithm;
@@ -74,7 +71,6 @@ public class TokenService {
return generateToken(UserWithPassword.convert(appUser), request);
}
public Optional<Claims> getClaims(HttpServletRequest request) {
String token = request.getHeader(authenticationConfig.getTokenHttpHeaderKey());
String appKey = getAppKey(request);
@@ -94,14 +90,6 @@ public class TokenService {
public Optional<Claims> getClaims(String token, String appKey) {
try {
if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌,需校验数据库是否存在该配置
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
UserTokenDO dbToken =
userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length()));
if (dbToken == null || !dbToken.getToken().equals(token.replace("Bearer ", ""))) {
throw new AccessException("Token does not exist :" + appKey);
}
}
String tokenSecret = getTokenSecret(appKey);
Claims claims =
Jwts.parser().setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8))
@@ -134,16 +122,6 @@ public class TokenService {
Map<String, String> appKeyToSecretMap = authenticationConfig.getAppKeyToSecretMap();
String secret = appKeyToSecretMap.get(appKey);
if (StringUtils.isBlank(secret)) {
if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌
String realAppKey = appKey.substring("SysDbToken:".length());
String tmp =
"WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==";
if (tmp.length() <= realAppKey.length()) {
return realAppKey;
} else {
return realAppKey + tmp.substring(realAppKey.length());
}
}
throw new AccessException("get secret from appKey failed :" + appKey);
}
return secret;

View File

@@ -75,12 +75,8 @@ public class SqlExecutor implements ChatQueryExecutor {
return null;
}
// 使用querySQL它已经包含了所有修正包括物理SQL修正
String finalSql = StringUtils.isNotBlank(parseInfo.getSqlInfo().getQuerySQL())
? parseInfo.getSqlInfo().getQuerySQL()
: parseInfo.getSqlInfo().getCorrectedS2SQL();
QuerySqlReq sqlReq = QuerySqlReq.builder().sql(finalSql).build();
QuerySqlReq sqlReq =
QuerySqlReq.builder().sql(parseInfo.getSqlInfo().getCorrectedS2SQL()).build();
sqlReq.setSqlInfo(parseInfo.getSqlInfo());
sqlReq.setDataSetId(parseInfo.getDataSetId());
@@ -94,7 +90,7 @@ public class SqlExecutor implements ChatQueryExecutor {
queryResult.setQueryTimeCost(System.currentTimeMillis() - startTime);
if (queryResp != null) {
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
queryResult.setQuerySql(finalSql);
queryResult.setQuerySql(queryResp.getSql());
queryResult.setQueryResults(queryResp.getResultList());
queryResult.setQueryColumns(queryResp.getColumns());
queryResult.setQueryState(QueryState.SUCCESS);

View File

@@ -47,8 +47,7 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
Agent agent = executeContext.getAgent();
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
return Objects.nonNull(chatApp) && chatApp.isEnable()
&& StringUtils.isNotBlank(executeContext.getResponse().getTextResult()) // 如果都没结果,则无法处理
&& StringUtils.isBlank(executeContext.getResponse().getTextSummary()); // 如果已经有汇总的结果了,无法再次处理
&& StringUtils.isNotBlank(executeContext.getResponse().getTextResult()); // 如果都没结果,则无法处理,直接跳过
}
@Override
@@ -58,16 +57,7 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
Map<String, Object> variable = new HashMap<>();
String question = executeContext.getResponse().getTextResult();// 结果解析应该用改写的问题,因为改写的内容信息量更大
if (executeContext.getParseInfo().getProperties() != null
&& executeContext.getParseInfo().getProperties().containsKey("CONTEXT")) {
Map<String, Object> context = (Map<String, Object>) executeContext.getParseInfo()
.getProperties().get("CONTEXT");
if (context.get("queryText") != null && "".equals(context.get("queryText"))) {
question = context.get("queryText").toString();
}
}
variable.put("question", question);
variable.put("question", executeContext.getRequest().getQueryText());
variable.put("data", queryResult.getTextResult());
Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variable);

View File

@@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -40,7 +39,6 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
private MemoryService memoryService;
@Autowired
@Lazy
private ChatQueryService chatQueryService;
@Autowired

View File

@@ -49,7 +49,6 @@ import net.sf.jsqlparser.schema.Column;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -67,7 +66,6 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired
@Lazy
private AgentService agentService;
private final List<ChatQueryParser> chatQueryParsers = ComponentFactory.getChatParsers();

View File

@@ -21,8 +21,7 @@ public class LoadRemoveService {
List<String> resultList = new ArrayList<>(value);
if (!CollectionUtils.isEmpty(modelIdOrDataSetIds)) {
resultList.removeIf(nature -> {
if (Objects.isNull(nature) || !nature.startsWith("_")) { // 系统的字典是以 _ 开头的,
// 过滤因引用外部字典导致的异常
if (Objects.isNull(nature)) {
return false;
}
Long id = getId(nature);

View File

@@ -22,6 +22,4 @@ public class Text2SQLExemplar implements Serializable {
private String dbSchema;
private String sql;
protected double similarity; // 传递相似度,可以作为样本筛选的依据
}

View File

@@ -72,10 +72,7 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner {
embeddingService.retrieveQuery(collection, retrieveQuery, num);
results.forEach(ret -> {
ret.getRetrieval().forEach(r -> {
Text2SQLExemplar tmp = // 传递相似度,可以作为样本筛选的依据
JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class);
tmp.setSimilarity(r.getSimilarity());
exemplars.add(tmp);
exemplars.add(JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class));
});
});

View File

@@ -18,7 +18,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_DETAIL_LIMIT;
import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_METRIC_LIMIT;
@@ -66,23 +65,12 @@ public class SemanticParseInfo implements Serializable {
DataSetMatchResult mr2 = getDataSetMatchResult(o2.getElementMatches());
double difference = mr1.getMaxDatesetSimilarity() - mr2.getMaxDatesetSimilarity();
if (Math.abs(difference) < 0.0005) { // 看完全匹配的个数,实践证明,可以用户输入规范后,该逻辑具有优势
if (!o1.getDataSetId().equals(o2.getDataSetId())) {
List<SchemaElementMatch> elementMatches1 = o1.getElementMatches().stream()
.filter(e -> e.getSimilarity() == 1).collect(Collectors.toList());
List<SchemaElementMatch> elementMatches2 = o2.getElementMatches().stream()
.filter(e -> e.getSimilarity() == 1).collect(Collectors.toList());
if (elementMatches1.size() > elementMatches2.size()) {
return -1;
} else if (elementMatches1.size() < elementMatches2.size()) {
return 1;
}
}
if (difference == 0) {
difference = mr1.getMaxMetricSimilarity() - mr2.getMaxMetricSimilarity();
if (Math.abs(difference) < 0.0005) {
if (difference == 0) {
difference = mr1.getTotalSimilarity() - mr2.getTotalSimilarity();
}
if (Math.abs(difference) < 0.0005) {
if (difference == 0) {
difference = mr1.getMaxMetricUseCnt() - mr2.getMaxMetricUseCnt();
}
}

View File

@@ -16,7 +16,4 @@ public class SqlInfo implements Serializable {
// SQL to be executed finally
private String querySQL;
// Physical SQL corrected by LLM for performance optimization
private String correctedQuerySQL;
}

View File

@@ -8,6 +8,5 @@ public enum ChatWorkflowState {
VALIDATING,
SQL_CORRECTING,
PROCESSING,
PHYSICAL_SQL_CORRECTING,
FINISHED
}

View File

@@ -1,98 +0,0 @@
package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.pojo.ChatApp;
import com.tencent.supersonic.common.pojo.enums.AppModule;
import com.tencent.supersonic.common.util.ChatAppManager;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import dev.langchain4j.model.chat.ChatLanguageModel;
import dev.langchain4j.model.input.Prompt;
import dev.langchain4j.model.input.PromptTemplate;
import dev.langchain4j.model.output.structured.Description;
import dev.langchain4j.provider.ModelProvider;
import dev.langchain4j.service.AiServices;
import lombok.Data;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
* 物理SQL修正器 - 使用LLM优化物理SQL性能
*/
@Slf4j
public class LLMPhysicalSqlCorrector extends BaseSemanticCorrector {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
public static final String APP_KEY = "PHYSICAL_SQL_CORRECTOR";
private static final String INSTRUCTION = ""
+ "#Role: You are a senior database performance optimization expert experienced in SQL tuning."
+ "\n\n#Task: You will be provided with a user question and the corresponding physical SQL query,"
+ " please analyze and optimize this SQL to improve query performance." + "\n\n#Rules:"
+ "\n1. DO NOT add or introduce any new fields, columns, or aliases that are not in the original SQL."
+ "\n2. Push WHERE conditions into JOIN ON clauses when possible to reduce intermediate result sets."
+ "\n3. Optimize JOIN order by placing smaller tables or tables with selective conditions first."
+ "\n4. For date range conditions, ensure they are applied as early as possible in the query execution."
+ "\n5. Remove or comment out database-specific index hints (like USE INDEX) that may cause syntax errors."
+ "\n6. ONLY modify the structure and order of existing elements, do not change field names or add new ones."
+ "\n7. Ensure the optimized SQL is syntactically correct and logically equivalent to the original."
+ "\n\n#Question: {{question}}" + "\n\n#OriginalSQL: {{sql}}";
public LLMPhysicalSqlCorrector() {
ChatAppManager.register(APP_KEY, ChatApp.builder().prompt(INSTRUCTION).name("物理SQL修正")
.appModule(AppModule.CHAT).description("通过大模型对物理SQL做性能优化").enable(false).build());
}
@Data
@ToString
static class PhysicalSql {
@Description("either positive or negative")
private String opinion;
@Description("optimized sql if negative")
private String sql;
}
interface PhysicalSqlExtractor {
PhysicalSql generatePhysicalSql(String text);
}
@Override
public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
ChatApp chatApp = chatQueryContext.getRequest().getChatAppConfig().get(APP_KEY);
if (!chatQueryContext.getRequest().getText2SQLType().enableLLM() || Objects.isNull(chatApp)
|| !chatApp.isEnable()) {
return;
}
ChatLanguageModel chatLanguageModel =
ModelProvider.getChatModel(chatApp.getChatModelConfig());
PhysicalSqlExtractor extractor =
AiServices.create(PhysicalSqlExtractor.class, chatLanguageModel);
Prompt prompt = generatePrompt(chatQueryContext.getRequest().getQueryText(),
semanticParseInfo, chatApp.getPrompt());
PhysicalSql physicalSql =
extractor.generatePhysicalSql(prompt.toUserMessage().singleText());
keyPipelineLog.info("LLMPhysicalSqlCorrector modelReq:\n{} \nmodelResp:\n{}", prompt.text(),
physicalSql);
if ("NEGATIVE".equalsIgnoreCase(physicalSql.getOpinion())
&& StringUtils.isNotBlank(physicalSql.getSql())) {
semanticParseInfo.getSqlInfo().setCorrectedQuerySQL(physicalSql.getSql());
}
}
private Prompt generatePrompt(String queryText, SemanticParseInfo semanticParseInfo,
String promptTemplate) {
Map<String, Object> variable = new HashMap<>();
variable.put("question", queryText);
variable.put("sql", semanticParseInfo.getSqlInfo().getQuerySQL());
return PromptTemplate.from(promptTemplate).apply(variable);
}
}

View File

@@ -14,8 +14,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.headless.chat.parser.ParserConfig.*;
@@ -49,33 +51,13 @@ public class PromptHelper {
// use random collection of exemplars for each self-consistency inference
for (int i = 0; i < selfConsistencyNumber; i++) {
List<Text2SQLExemplar> shuffledList = new ArrayList<>(exemplars);
List<Text2SQLExemplar> same = shuffledList.stream() // 相似度极高的话,先找出来
.filter(e -> e.getSimilarity() > 0.989).collect(Collectors.toList());
List<Text2SQLExemplar> noSame = shuffledList.stream()
.filter(e -> e.getSimilarity() <= 0.989).collect(Collectors.toList());
if ((noSame.size() - same.size()) > fewShotNumber) {// 去除部分最低分
noSame.sort(Comparator.comparingDouble(Text2SQLExemplar::getSimilarity));
noSame = noSame.subList((noSame.size() - fewShotNumber) / 2, noSame.size());
}
Text2SQLExemplar mostSimilar = noSame.get(noSame.size() - 1);
Collections.shuffle(noSame);
List<Text2SQLExemplar> ts;
if (same.size() > 0) {// 一样的话,必须作为提示语
ts = new ArrayList<>();
int needSize = Math.min(noSame.size() + same.size(), fewShotNumber);
if (needSize > same.size()) {
ts.addAll(noSame.subList(0, needSize - same.size()));
}
ts.addAll(same);
} else { // 至少要一个最像的
ts = noSame.subList(0, Math.min(noSame.size(), fewShotNumber));
if (!ts.contains(mostSimilar)) {
ts.remove(ts.size() - 1);
ts.add(mostSimilar);
}
}
results.add(ts);
// only shuffle the exemplars from config
List<Text2SQLExemplar> subList =
shuffledList.subList(llmReq.getDynamicExemplars().size(), shuffledList.size());
Collections.shuffle(subList);
results.add(shuffledList.subList(0, Math.min(shuffledList.size(), fewShotNumber)));
}
return results;
}

View File

@@ -8,7 +8,6 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.corrector.LLMPhysicalSqlCorrector;
import com.tencent.supersonic.headless.chat.corrector.SemanticCorrector;
import com.tencent.supersonic.headless.chat.mapper.SchemaMapper;
import com.tencent.supersonic.headless.chat.parser.SemanticParser;
@@ -77,10 +76,6 @@ public class ChatWorkflowEngine {
long start = System.currentTimeMillis();
performTranslating(queryCtx, parseResult);
parseResult.getParseTimeCost().setSqlTime(System.currentTimeMillis() - start);
queryCtx.setChatWorkflowState(ChatWorkflowState.PHYSICAL_SQL_CORRECTING);
break;
case PHYSICAL_SQL_CORRECTING:
performPhysicalSqlCorrecting(queryCtx);
queryCtx.setChatWorkflowState(ChatWorkflowState.FINISHED);
break;
default:
@@ -167,25 +162,4 @@ public class ChatWorkflowEngine {
parseResult.setErrorMsg(String.join("\n", errorMsg));
}
}
private void performPhysicalSqlCorrecting(ChatQueryContext queryCtx) {
List<SemanticQuery> candidateQueries = queryCtx.getCandidateQueries();
if (CollectionUtils.isNotEmpty(candidateQueries)) {
for (SemanticQuery semanticQuery : candidateQueries) {
for (SemanticCorrector corrector : semanticCorrectors) {
if (corrector instanceof LLMPhysicalSqlCorrector) {
corrector.correct(queryCtx, semanticQuery.getParseInfo());
// 如果物理SQL被修正了更新querySQL为修正后的版本
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
if (StringUtils.isNotBlank(parseInfo.getSqlInfo().getCorrectedQuerySQL())) {
parseInfo.getSqlInfo().setQuerySQL(parseInfo.getSqlInfo().getCorrectedQuerySQL());
log.info("Physical SQL corrected and updated querySQL: {}",
parseInfo.getSqlInfo().getQuerySQL());
}
break;
}
}
}
}
}
}

View File

@@ -109,7 +109,7 @@ public class QueryUtils {
column.setModelId(metric.getModelId());
}
// if column nameEn contains metric alias, use metric dataFormatType
if (column.getDataFormatType() == null && StringUtils.isNotEmpty(metric.getAlias())) {
if (column.getDataFormatType() == null && metric.getAlias() != null) {
for (String alias : metric.getAlias().split(",")) {
if (nameEn.contains(alias)) {
column.setDataFormatType(metric.getDataFormatType());

View File

@@ -14,8 +14,7 @@ com.tencent.supersonic.headless.chat.parser.SemanticParser=\
com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\
com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector,\
com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector,\
com.tencent.supersonic.headless.chat.corrector.LLMPhysicalSqlCorrector
com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector
com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\
com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl

View File

@@ -15,8 +15,7 @@ com.tencent.supersonic.headless.chat.parser.SemanticParser=\
com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\
com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector,\
com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector,\
com.tencent.supersonic.headless.chat.corrector.LLMPhysicalSqlCorrector
com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector
com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\
com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl

10
pom.xml
View File

@@ -10,9 +10,9 @@
<version>${revision}</version>
<modules>
<module>common</module>
<module>auth</module>
<module>chat</module>
<module>common</module>
<module>launchers</module>
<module>headless</module>
</modules>
@@ -31,7 +31,6 @@
<java.target.version>21</java.target.version>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<maven.compiler.release>21</maven.compiler.release>
<file.encoding>UTF-8</file.encoding>
<jsqlparser.version>4.9</jsqlparser.version>
<pagehelper.version>6.1.0</pagehelper.version>
@@ -255,13 +254,6 @@
<source>${java.source.version}</source>
<target>${java.target.version}</target>
<encoding>${file.encoding}</encoding>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>

View File

@@ -1,5 +0,0 @@
{
"dependencies": {
"supersonic-chat-sdk": "link:packages/chat-sdk"
}
}

View File

@@ -79,7 +79,6 @@ export type SqlInfoType = {
parsedS2SQL: string;
correctedS2SQL: string;
querySQL: string;
correctedQuerySQL?: string;
};
export type ChatContextType = {

View File

@@ -58,28 +58,32 @@ const SqlItem: React.FC<Props> = ({
const getSchemaMapText = () => {
return `
Schema映射
${schema?.fieldNameList?.length > 0 ? `名称:${schema.fieldNameList.join('、')}` : ''}${schema?.values?.length > 0
${schema?.fieldNameList?.length > 0 ? `名称:${schema.fieldNameList.join('、')}` : ''}${
schema?.values?.length > 0
? `
取值:${schema.values
.map((item: any) => {
return `${item.fieldName}: ${item.fieldValue}`;
})
.join('、')}`
.map((item: any) => {
return `${item.fieldName}: ${item.fieldValue}`;
})
.join('、')}`
: ''
}${priorExts
}${
priorExts
? `
附加:${priorExts}`
: ''
}${terms?.length > 0
}${
terms?.length > 0
? `
术语:${terms
.map((item: any) => {
return `${item.name}${item.alias?.length > 0 ? `(${item.alias.join(',')})` : ''}: ${item.description
.map((item: any) => {
return `${item.name}${item.alias?.length > 0 ? `(${item.alias.join(',')})` : ''}: ${
item.description
}`;
})
.join('、')}`
})
.join('、')}`
: ''
}
}
`;
};
@@ -87,16 +91,16 @@ ${schema?.fieldNameList?.length > 0 ? `名称:${schema.fieldNameList.join('、
const getFewShotText = () => {
return `
Few-shot示例${fewShots
.map((item: any, index: number) => {
return `
.map((item: any, index: number) => {
return `
示例${index + 1}
问题:${item.question}
SQL
${format(item.sql)}
`;
})
.join('')}
})
.join('')}
`;
};
@@ -116,14 +120,6 @@ ${format(sqlInfo.correctedS2SQL)}
`;
};
const getCorrectedQuerySQLText = () => {
return `
物理SQL修正
${format(sqlInfo.correctedQuerySQL || '')}
`;
};
const getQuerySQLText = () => {
return `
最终执行SQL
@@ -159,9 +155,6 @@ ${executeErrorMsg}
if (sqlInfo.correctedS2SQL) {
text += getCorrectedS2SQLText();
}
if (sqlInfo.correctedQuerySQL) {
text += getCorrectedQuerySQLText();
}
if (sqlInfo.querySQL) {
text += getQuerySQLText();
}
@@ -190,8 +183,9 @@ ${executeErrorMsg}
<div className={`${tipPrefixCls}-content-options`}>
{llmReq && (
<div
className={`${tipPrefixCls}-content-option ${sqlType === 'schemaMap' ? `${tipPrefixCls}-content-option-active` : ''
}`}
className={`${tipPrefixCls}-content-option ${
sqlType === 'schemaMap' ? `${tipPrefixCls}-content-option-active` : ''
}`}
onClick={() => {
setSqlType(sqlType === 'schemaMap' ? '' : 'schemaMap');
}}
@@ -201,8 +195,9 @@ ${executeErrorMsg}
)}
{fewShots.length > 0 && (
<div
className={`${tipPrefixCls}-content-option ${sqlType === 'fewShots' ? `${tipPrefixCls}-content-option-active` : ''
}`}
className={`${tipPrefixCls}-content-option ${
sqlType === 'fewShots' ? `${tipPrefixCls}-content-option-active` : ''
}`}
onClick={() => {
setSqlType(sqlType === 'fewShots' ? '' : 'fewShots');
}}
@@ -212,8 +207,9 @@ ${executeErrorMsg}
)}
{sqlInfo.parsedS2SQL && (
<div
className={`${tipPrefixCls}-content-option ${sqlType === 'parsedS2SQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
className={`${tipPrefixCls}-content-option ${
sqlType === 'parsedS2SQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
onClick={() => {
setSqlType(sqlType === 'parsedS2SQL' ? '' : 'parsedS2SQL');
}}
@@ -223,8 +219,9 @@ ${executeErrorMsg}
)}
{sqlInfo.correctedS2SQL && (
<div
className={`${tipPrefixCls}-content-option ${sqlType === 'correctedS2SQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
className={`${tipPrefixCls}-content-option ${
sqlType === 'correctedS2SQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
onClick={() => {
setSqlType(sqlType === 'correctedS2SQL' ? '' : 'correctedS2SQL');
}}
@@ -232,26 +229,16 @@ ${executeErrorMsg}
S2SQL
</div>
)}
{sqlInfo.correctedQuerySQL && (
<div
className={`${tipPrefixCls}-content-option ${sqlType === 'correctedQuerySQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
onClick={() => {
setSqlType(sqlType === 'correctedQuerySQL' ? '' : 'correctedQuerySQL');
}}
>
SQL修正
</div>
)}
{sqlInfo.querySQL && (
<div
className={`${tipPrefixCls}-content-option ${sqlType === 'querySQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
className={`${tipPrefixCls}-content-option ${
sqlType === 'querySQL' ? `${tipPrefixCls}-content-option-active` : ''
}`}
onClick={() => {
setSqlType(sqlType === 'querySQL' ? '' : 'querySQL');
}}
>
{sqlInfo.correctedQuerySQL ? '最终执行SQL' : '最终执行SQL'}
SQL
</div>
)}
<Button className={`${prefixCls}-export-log`} size="small" onClick={onExportLog}>
@@ -261,12 +248,13 @@ ${executeErrorMsg}
</div>
</div>
<div
className={`${prefixCls} ${!window.location.pathname.includes('/chat') &&
className={`${prefixCls} ${
!window.location.pathname.includes('/chat') &&
integrateSystem &&
integrateSystem !== 'wiki'
? `${prefixCls}-copilot`
: ''
}`}
? `${prefixCls}-copilot`
: ''
}`}
>
{sqlType === 'schemaMap' && (
<div className={`${prefixCls}-code`}>
@@ -302,8 +290,9 @@ ${executeErrorMsg}
<div className={`${prefixCls}-schema-content`}>
{terms
.map((item: any) => {
return `${item.name}${item.alias?.length > 0 ? `(${item.alias.join(',')})` : ''
}: ${item.description}`;
return `${item.name}${
item.alias?.length > 0 ? `(${item.alias.join(',')})` : ''
}: ${item.description}`;
})
.join('、')}
</div>