topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-15 14:36:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(launcher)Refactor config keys, use s2 as the prefix

Browse Source
This commit is contained in:
jerryjzhang
2024-05-25 01:36:24 +08:00
parent 6745df5a2e
commit 822879cd7b
10 changed files with 111 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/MultiTurnParser.java
View File

@@ -53,7 +53,7 @@ public class MultiTurnParser implements ChatParser {
public void parse(ChatParseContext chatParseContext, ParseResp parseResp) { public void parse(ChatParseContext chatParseContext, ParseResp parseResp) {
Environment environment = ContextUtils.getBean(Environment.class); Environment environment = ContextUtils.getBean(Environment.class);
MultiTurnConfig agentMultiTurnConfig = chatParseContext.getAgent().getMultiTurnConfig(); MultiTurnConfig agentMultiTurnConfig = chatParseContext.getAgent().getMultiTurnConfig();
Boolean globalMultiTurnConfig = environment.getProperty("s2.multi-turn.enable", Boolean.class); Boolean globalMultiTurnConfig = environment.getProperty("s2.parser.multi-turn.enable", Boolean.class);
Boolean multiTurnConfig = agentMultiTurnConfig != null Boolean multiTurnConfig = agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn() : globalMultiTurnConfig; ? agentMultiTurnConfig.isEnableMultiTurn() : globalMultiTurnConfig;

24
common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java
View File

@@ -8,37 +8,37 @@ import org.springframework.context.annotation.Configuration;
@Data @Data
public class EmbeddingConfig { public class EmbeddingConfig {
@Value("${embedding.url:}") @Value("${s2.embedding.url:}")
private String url; private String url;
@Value("${embedding.recognize.path:/preset_query_retrival}") @Value("${s2.embedding.persistent.path:/tmp}")
private String embeddingStorePersistentPath;
@Value("${s2.embedding.recognize.path:/preset_query_retrival}")
private String recognizePath; private String recognizePath;
@Value("${embedding.preset.collection:preset_query_collection}") @Value("${s2.embedding.preset.collection:preset_query_collection}")
private String presetCollection; private String presetCollection;
@Value("${embedding.meta.collection:meta_collection}") @Value("${s2.embedding.meta.collection:meta_collection}")
private String metaCollectionName; private String metaCollectionName;
@Value("${embedding.nResult:1}") @Value("${s2.embedding.nResult:1}")
private int nResult; private int nResult;
@Value("${embedding.solved.query.collection:solved_query_collection}") @Value("${s2.embedding.solved.query.collection:solved_query_collection}")
private String solvedQueryCollection; private String solvedQueryCollection;
@Value("${embedding.solved.query.nResult:5}") @Value("${s2.embedding.solved.query.nResult:5}")
private int solvedQueryResultNum; private int solvedQueryResultNum;
@Value("${embedding.metric.analyzeQuery.collection:solved_query_collection}") @Value("${s2.embedding.metric.analyzeQuery.collection:solved_query_collection}")
private String metricAnalyzeQueryCollection; private String metricAnalyzeQueryCollection;
@Value("${text2sql.collection.name:text2dsl_agent_collection}") @Value("${text2sql.collection.name:text2dsl_agent_collection}")
private String text2sqlCollectionName; private String text2sqlCollectionName;
@Value("${embedding.metric.analyzeQuery.nResult:5}") @Value("${s2.embedding.metric.analyzeQuery.nResult:5}")
private int metricAnalyzeQueryResultNum; private int metricAnalyzeQueryResultNum;
@Value("${inMemoryEmbeddingStore.persistent.path:/tmp}")
private String embeddingStorePersistentPath;
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/BaseSemanticCorrector.java
View File

@@ -82,7 +82,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
//decide whether add order by expression field to select //decide whether add order by expression field to select
Environment environment = ContextUtils.getBean(Environment.class); Environment environment = ContextUtils.getBean(Environment.class);
String correctorAdditionalInfo = environment.getProperty("corrector.additional.information"); String correctorAdditionalInfo = environment.getProperty("s2.corrector.additional.information");
if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) { if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) {
needAddFields.addAll(SqlSelectHelper.getOrderByFields(correctS2SQL)); needAddFields.addAll(SqlSelectHelper.getOrderByFields(correctS2SQL));
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GroupByCorrector.java
View File

@@ -57,7 +57,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
return false; return false;
} }
Environment environment = ContextUtils.getBean(Environment.class); Environment environment = ContextUtils.getBean(Environment.class);
String correctorAdditionalInfo = environment.getProperty("corrector.additional.information"); String correctorAdditionalInfo = environment.getProperty("s2.corrector.additional.information");
if (StringUtils.isNotBlank(correctorAdditionalInfo) && !Boolean.parseBoolean(correctorAdditionalInfo)) { if (StringUtils.isNotBlank(correctorAdditionalInfo) && !Boolean.parseBoolean(correctorAdditionalInfo)) {
return false; return false;
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/HavingCorrector.java
View File

@@ -31,7 +31,7 @@ public class HavingCorrector extends BaseSemanticCorrector {
//decide whether add having expression field to select //decide whether add having expression field to select
Environment environment = ContextUtils.getBean(Environment.class); Environment environment = ContextUtils.getBean(Environment.class);
String correctorAdditionalInfo = environment.getProperty("corrector.additional.information"); String correctorAdditionalInfo = environment.getProperty("s2.corrector.additional.information");
if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) { if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) {
addHavingToSelect(semanticParseInfo); addHavingToSelect(semanticParseInfo);
} }

13
headless/core/src/main/java/com/tencent/supersonic/headless/core/config/LLMParserConfig.java
View File

@@ -9,22 +9,21 @@ import org.springframework.context.annotation.Configuration;
@Data @Data
public class LLMParserConfig { public class LLMParserConfig {
@Value("${s2.parser.url:}")
@Value("${llm.parser.url:}")
private String url; private String url;
@Value("${query2sql.path:/query2sql}") @Value("${s2.query2sql.path:/query2sql}")
private String queryToSqlPath; private String queryToSqlPath;
@Value("${dimension.topn:10}") @Value("${s2.dimension.topn:10}")
private Integer dimensionTopN; private Integer dimensionTopN;
@Value("${metric.topn:10}") @Value("${s2.metric.topn:10}")
private Integer metricTopN; private Integer metricTopN;
@Value("${tag.topn:20}") @Value("${s2.tag.topn:20}")
private Integer tagTopN; private Integer tagTopN;
@Value("${all.model:false}") @Value("${s2.all.model:false}")
private Boolean allModel; private Boolean allModel;
} }

47
headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java
View File

@@ -14,72 +14,73 @@ import org.springframework.context.annotation.Configuration;
@Slf4j @Slf4j
public class OptimizationConfig { public class OptimizationConfig {
@Value("${one.detection.size:8}") @Value("${s2.one.detection.size:8}")
private Integer oneDetectionSize; private Integer oneDetectionSize;
@Value("${one.detection.max.size:20}") @Value("${s2.one.detection.max.size:20}")
private Integer oneDetectionMaxSize; private Integer oneDetectionMaxSize;
@Value("${metric.dimension.min.threshold:0.3}") @Value("${s2.metric.dimension.min.threshold:0.3}")
private Double metricDimensionMinThresholdConfig; private Double metricDimensionMinThresholdConfig;
@Value("${metric.dimension.threshold:0.3}") @Value("${s2.metric.dimension.threshold:0.3}")
private Double metricDimensionThresholdConfig; private Double metricDimensionThresholdConfig;
@Value("${dimension.value.min.threshold:0.2}") @Value("${s2.dimension.value.min.threshold:0.2}")
private Double dimensionValueMinThresholdConfig; private Double dimensionValueMinThresholdConfig;
@Value("${dimension.value.threshold:0.5}") @Value("${s2.dimension.value.threshold:0.5}")
private Double dimensionValueThresholdConfig; private Double dimensionValueThresholdConfig;
@Value("${long.text.threshold:0.8}") @Value("${s2.long.text.threshold:0.8}")
private Double longTextThreshold; private Double longTextThreshold;
@Value("${short.text.threshold:0.5}") @Value("${s2.short.text.threshold:0.5}")
private Double shortTextThreshold; private Double shortTextThreshold;
@Value("${query.text.length.threshold:10}") @Value("${s2.query.text.length.threshold:10}")
private Integer queryTextLengthThreshold; private Integer queryTextLengthThreshold;
@Value("${embedding.mapper.word.min:4}")
@Value("${s2.embedding.mapper.word.min:4}")
private int embeddingMapperWordMin; private int embeddingMapperWordMin;
@Value("${embedding.mapper.word.max:4}") @Value("${s2.embedding.mapper.word.max:4}")
private int embeddingMapperWordMax; private int embeddingMapperWordMax;
@Value("${embedding.mapper.batch:50}") @Value("${s2.embedding.mapper.batch:50}")
private int embeddingMapperBatch; private int embeddingMapperBatch;
@Value("${embedding.mapper.number:5}") @Value("${s2.embedding.mapper.number:5}")
private int embeddingMapperNumber; private int embeddingMapperNumber;
@Value("${embedding.mapper.round.number:10}") @Value("${s2.embedding.mapper.round.number:10}")
private int embeddingMapperRoundNumber; private int embeddingMapperRoundNumber;
@Value("${embedding.mapper.min.threshold:0.6}") @Value("${s2.embedding.mapper.min.threshold:0.6}")
private Double embeddingMapperMinThreshold; private Double embeddingMapperMinThreshold;
@Value("${embedding.mapper.threshold:0.99}") @Value("${s2.embedding.mapper.threshold:0.99}")
private Double embeddingMapperThreshold; private Double embeddingMapperThreshold;
@Value("${s2SQL.linking.value.switch:true}") @Value("${s2.parser.linking.value.switch:true}")
private boolean useLinkingValueSwitch; private boolean useLinkingValueSwitch;
@Value("${s2SQL.generation:TWO_PASS_AUTO_COT}") @Value("${s2.parser.generation:TWO_PASS_AUTO_COT}")
private LLMReq.SqlGenType sqlGenType; private LLMReq.SqlGenType sqlGenType;
@Value("${s2SQL.use.switch:true}") @Value("${s2.parser.use.switch:true}")
private boolean useS2SqlSwitch; private boolean useS2SqlSwitch;
@Value("${text2sql.example.num:15}") @Value("${s2.parser.exemplar-recall.num:15}")
private int text2sqlExampleNum; private int text2sqlExampleNum;
@Value("${text2sql.fewShots.num:10}") @Value("${s2.parser.few-shot.num:10}")
private int text2sqlFewShotsNum; private int text2sqlFewShotsNum;
@Value("${text2sql.self.consistency.num:5}") @Value("${s2.parser.self-consistency.num:5}")
private int text2sqlSelfConsistencyNum; private int text2sqlSelfConsistencyNum;
@Value("${parse.show.count:3}") @Value("${s2.parser.show-count:3}")
private Integer parseShowCount; private Integer parseShowCount;
@Autowired @Autowired

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -42,7 +42,7 @@ public class QueryUtils {
private final Set<Pattern> patterns = new HashSet<>(); private final Set<Pattern> patterns = new HashSet<>();
@Value("${query.optimizer.enable:true}") @Value("${s2.query-optimizer.enable:true}")
private Boolean optimizeEnable; private Boolean optimizeEnable;
@PostConstruct @PostConstruct

74
launchers/standalone/src/main/resources/application-local.yaml
View File

@@ -30,44 +30,42 @@ authentication:
header: header:
key: Authorization key: Authorization
query: logging:
optimizer: level:
enable: true dev.langchain4j: DEBUG
dev.ai4j.openai4j: DEBUG
time:
threshold: 100
dimension:
topn: 20
metric:
topn: 20
corrector:
additional:
information: true
pyllm:
url: http://127.0.0.1:9092
llm:
parser:
url: ${pyllm.url}
embedding:
url: ${pyllm.url}
functionCall:
url: ${pyllm.url}
text2sql:
example:
num: 1
s2: s2:
pyllm:
url: http://127.0.0.1:9092
parser:
url: ${s2.pyllm.url}
exemplar-recall:
number: 10
few-shot:
number: 5
self-consistency:
number: 5
multi-turn:
enable: false
corrector:
additional:
information: true
functionCall:
url: ${s2.pyllm.url}
embedding:
url: ${s2.pyllm.url}
persistent:
path: /tmp
demo: demo:
names: S2VisitsDemo,S2ArtistDemo names: S2VisitsDemo,S2ArtistDemo
enableLLM: true enableLLM: true
multi-turn:
enable: false
langchain4j: langchain4j:
#1.chat-model #1.chat-model
chat-model: chat-model:
@@ -103,13 +101,3 @@ s2:
# access-token: hg_access_token # access-token: hg_access_token
# model-id: sentence-transformers/all-MiniLM-L6-v2 # model-id: sentence-transformers/all-MiniLM-L6-v2
# timeout: 1h # timeout: 1h
#langchain4j log
logging:
level:
dev.langchain4j: DEBUG
dev.ai4j.openai4j: DEBUG
inMemoryEmbeddingStore:
persistent:
path: /tmp

74
launchers/standalone/src/test/resources/application-local.yaml
View File

@@ -30,44 +30,42 @@ authentication:
header: header:
key: Authorization key: Authorization
query: logging:
optimizer: level:
enable: true dev.langchain4j: DEBUG
dev.ai4j.openai4j: DEBUG
time:
threshold: 100
dimension:
topn: 20
metric:
topn: 20
corrector:
additional:
information: true
pyllm:
url: http://127.0.0.1:9092
llm:
parser:
url: ${pyllm.url}
embedding:
url: ${pyllm.url}
functionCall:
url: ${pyllm.url}
text2sql:
example:
num: 1
s2: s2:
pyllm:
url: http://127.0.0.1:9092
parser:
url: ${s2.pyllm.url}
exemplar-recall:
number: 10
few-shot:
number: 5
self-consistency:
number: 5
multi-turn:
enable: false
corrector:
additional:
information: true
functionCall:
url: ${s2.pyllm.url}
embedding:
url: ${s2.pyllm.url}
persistent:
path: /tmp
demo: demo:
names: S2VisitsDemo,S2ArtistDemo names: S2VisitsDemo,S2ArtistDemo
enableLLM: true enableLLM: true
multi-turn:
enable: false
langchain4j: langchain4j:
#1.chat-model #1.chat-model
chat-model: chat-model:
@@ -103,13 +101,3 @@ s2:
# access-token: hg_access_token # access-token: hg_access_token
# model-id: sentence-transformers/all-MiniLM-L6-v2 # model-id: sentence-transformers/all-MiniLM-L6-v2
# timeout: 1h # timeout: 1h
#langchain4j log
logging:
level:
dev.langchain4j: DEBUG
dev.ai4j.openai4j: DEBUG
inMemoryEmbeddingStore:
persistent:
path: /tmp