[improvement](chat) Change llmparser to pyllm, retrieve LLMProxy from environment variables, defaulting to JavaLLMProxy. (#497)

assembly/bin/supersonic-common.sh

@@ -11,14 +11,14 @@ buildDir=$baseDir/build
 
 readonly CHAT_APP_NAME="supersonic_chat"
 readonly SEMANTIC_APP_NAME="supersonic_semantic"
-readonly LLMPARSER_APP_NAME="supersonic_llmparser"
+readonly PYLLM_APP_NAME="supersonic_pyllm"
 readonly STANDALONE_APP_NAME="supersonic_standalone"
 readonly CHAT_SERVICE="chat"
 readonly SEMANTIC_SERVICE="semantic"
-readonly LLMPARSER_SERVICE="llmparser"
+readonly PYLLM_SERVICE="pyllm"
 readonly STANDALONE_SERVICE="standalone"
-readonly LLMPARSER_HOST="127.0.0.1"
-readonly LLMPARSER_PORT="9092"
+readonly PYLLM_HOST="127.0.0.1"
+readonly PYLLM_PORT="9092"
 
 function setEnvToWeb {
    model_name=$1
@@ -81,23 +81,23 @@ function runJavaService {
 
 # run python service
 function runPythonService {
-  pythonRunDir=${runtimeDir}/supersonic-${model_name}/llmparser
+  pythonRunDir=${runtimeDir}/supersonic-${model_name}/pyllm
   cd $pythonRunDir
-  nohup ${python_path} supersonic_llmparser.py  > $pythonRunDir/llmparser.log  2>&1   &
+  nohup ${python_path} supersonic_pyllm.py  > $pythonRunDir/pyllm.log  2>&1   &
   # add health check
   for i in {1..10}
   do
-    echo "llmparser health check attempt $i..."
-    response=$(curl -s http://${LLMPARSER_HOST}:${LLMPARSER_PORT}/health)
-    echo "llmparser health check response: $response"
+    echo "pyllm health check attempt $i..."
+    response=$(curl -s http://${PYLLM_HOST}:${PYLLM_PORT}/health)
+    echo "pyllm health check response: $response"
     status_ok="Healthy"
     if [[ $response == *$status_ok* ]] ; then
-      echo "llmparser Health check passed."
+      echo "pyllm Health check passed."
       break
     else
       if [ "$i" -eq 10 ]; then
-        echo "llmparser Health check failed after 10 attempts."
-        echo "May still downloading model files. Please check llmparser.log in runtime directory."
+        echo "pyllm Health check failed after 10 attempts."
+        echo "May still downloading model files. Please check pyllm.log in runtime directory."
       fi
       echo "Retrying after 5 seconds..."
       sleep 5

assembly/bin/supersonic-daemon.bat

@@ -9,10 +9,10 @@ set "main_class=com.tencent.supersonic.StandaloneLauncher"
 set "python_path=python"
 set "pip_path=pip3"
 set "standalone_service=standalone"
-set "llmparser_service=llmparser"
+set "pyllm_service=pyllm"
 
 set "javaRunDir=%runtimeDir%\supersonic-standalone"
-set "pythonRunDir=%runtimeDir%\supersonic-standalone\llmparser"
+set "pythonRunDir=%runtimeDir%\supersonic-standalone\pyllm"
 
 set "command=%~1"
 set "service=%~2"
@@ -42,7 +42,7 @@ if "%command%"=="restart" (
 )
 
 :START
-    if "%service%"=="%llmparser_service%" (
+    if "%service%"=="%pyllm_service%" (
          call :START_PYTHON
          goto :EOF
     )
@@ -51,7 +51,7 @@ if "%command%"=="restart" (
     goto :EOF
 
 :STOP
-    if "%service%"=="%llmparser_service%" (
+    if "%service%"=="%pyllm_service%" (
         call :STOP_PYTHON
         goto :EOF
     )
@@ -60,9 +60,9 @@ if "%command%"=="restart" (
     goto :EOF
 
 :START_PYTHON
-   echo 'python service starting, see logs in llmparser/llmparser.log'
+   echo 'python service starting, see logs in pyllm/pyllm.log'
    cd "%pythonRunDir%"
-   start /B %python_path% supersonic_llmparser.py  > %pythonRunDir%\llmparser.log 2>&1
+   start /B %python_path% supersonic_pyllm.py  > %pythonRunDir%\pyllm.log 2>&1
    timeout /t 10 >nul
    echo 'python service started'
    goto :EOF
@@ -96,7 +96,7 @@ if "%command%"=="restart" (
    goto :EOF
 
 :RELOAD_EXAMPLE
-   cd "%runtimeDir%\supersonic-standalone\llmparser\sql"
+   cd "%runtimeDir%\supersonic-standalone\pyllm\sql"
    start  %python_path% examples_reload_run.py
    goto :EOF
 

assembly/bin/supersonic-daemon.sh

@@ -22,8 +22,9 @@ app_name=$STANDALONE_APP_NAME
 main_class="com.tencent.supersonic.StandaloneLauncher"
 model_name=$service
 
-if [ "$service" == "llmparser" ]; then
+if [ "$service" == "pyllm" ]; then
   model_name=${STANDALONE_SERVICE}
+  export llmProxy=PythonLLMProxy
 fi
 
 cd $baseDir
@@ -43,14 +44,14 @@ function setAppName {
     app_name=$CHAT_APP_NAME
   elif [ "$service" == $SEMANTIC_SERVICE ]; then
     app_name=$SEMANTIC_APP_NAME
-  elif [ "$service" == $LLMPARSER_SERVICE ]; then
-    app_name=$LLMPARSER_APP_NAME
+  elif [ "$service" == $PYLLM_SERVICE ]; then
+    app_name=$PYLLM_APP_NAME
   fi
 }
 setAppName
 
 function reloadExamples {
-  pythonRunDir=${runtimeDir}/supersonic-${model_name}/llmparser
+  pythonRunDir=${runtimeDir}/supersonic-${model_name}/pyllm
   cd $pythonRunDir/sql
   ${python_path} examples_reload_run.py
 }
@@ -61,7 +62,7 @@ function start()
   local_app_name=$1
   pid=$(ps aux |grep ${local_app_name} | grep -v grep | awk '{print $2}')
   if [[ "$pid" == "" ]]; then
-    if [[ ${local_app_name} == $LLMPARSER_APP_NAME ]]; then
+    if [[ ${local_app_name} == $PYLLM_APP_NAME ]]; then
       runPythonService ${local_app_name}
     else
       runJavaService ${local_app_name}
@@ -87,7 +88,7 @@ function stop()
 
 function reload()
 {
-  if [[ $1 == $LLMPARSER_APP_NAME ]]; then
+  if [[ $1 == $PYLLM_APP_NAME ]]; then
     reloadExamples
   fi
 }
@@ -95,11 +96,11 @@ function reload()
 # 4. execute command operation
 case "$command" in
   start)
-        if [ "$service" == $STANDALONE_SERVICE ]; then
-          echo  "Starting $LLMPARSER_APP_NAME"
-          start $LLMPARSER_APP_NAME
+        if [ "$service" == $PYLLM_SERVICE ]; then
           echo  "Starting $app_name"
           start $app_name
+          echo  "Starting $STANDALONE_APP_NAME"
+          start $STANDALONE_APP_NAME
         else
           echo  "Starting $app_name"
           start $app_name
@@ -107,11 +108,11 @@ case "$command" in
         echo  "Start success"
         ;;
   stop)
-        if [ "$service" == $STANDALONE_SERVICE ]; then
-          echo  "Stopping $LLMPARSER_APP_NAME"
-          stop $LLMPARSER_APP_NAME
+        if [ "$service" == $PYLLM_SERVICE ]; then
           echo  "Stopping $app_name"
           stop $app_name
+          echo  "Stopping $STANDALONE_APP_NAME"
+          stop $STANDALONE_APP_NAME
         else
           echo  "Stopping $app_name"
           stop ${app_name}
@@ -124,15 +125,15 @@ case "$command" in
         echo  "Reload success"
         ;;
   restart)
-        if [ "$service" == $STANDALONE_SERVICE ]; then
+        if [ "$service" == $PYLLM_SERVICE ]; then
           echo  "Stopping ${app_name}"
           stop ${app_name}
-          echo  "Stopping ${LLMPARSER_APP_NAME}"
-          stop $LLMPARSER_APP_NAME
-          echo  "Starting ${LLMPARSER_APP_NAME}"
-          start $LLMPARSER_APP_NAME
+          echo  "Stopping ${STANDALONE_APP_NAME}"
+          stop $STANDALONE_APP_NAME
           echo  "Starting ${app_name}"
           start ${app_name}
+          echo  "Starting ${STANDALONE_APP_NAME}"
+          start $STANDALONE_APP_NAME
         else
           echo  "Stopping ${app_name}"
           stop ${app_name}

assembly/build/build.xml

@@ -22,7 +22,7 @@
         </fileSet>
         <fileSet>
             <directory>${project.basedir}/../../chat/python</directory>
-            <outputDirectory>llmparser</outputDirectory>
+            <outputDirectory>pyllm</outputDirectory>
             <fileMode>0777</fileMode>
             <directoryMode>0755</directoryMode>
         </fileSet>

chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FromCorrector.java

@@ -4,7 +4,9 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
 import lombok.extern.slf4j.Slf4j;
-
+/**
+ * Perform SQL corrections on the "From" section in S2SQL.
+ */
 @Slf4j
 public class FromCorrector extends BaseSemanticCorrector {
 

chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java

@@ -18,7 +18,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
- * Perform SQL corrections on the "group by" section in S2SQL.
+ * Perform SQL corrections on the "Group by" section in S2SQL.
  */
 @Slf4j
 public class GroupByCorrector extends BaseSemanticCorrector {

chat/core/src/main/java/com/tencent/supersonic/chat/parser/JavaLLMProxy.java

@@ -13,11 +13,13 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import dev.langchain4j.model.chat.ChatLanguageModel;
 import java.util.Objects;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
 
 /**
  * LLMProxy based on langchain4j Java version.
  */
 @Slf4j
+@Component
 public class JavaLLMProxy implements LLMProxy {
 
     @Override

chat/core/src/main/java/com/tencent/supersonic/chat/parser/PythonLLMProxy.java

@@ -19,6 +19,7 @@ import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponentsBuilder;
 
@@ -26,6 +27,7 @@ import org.springframework.web.util.UriComponentsBuilder;
  * PythonLLMProxy sends requests to LangChain-based python service.
  */
 @Slf4j
+@Component
 public class PythonLLMProxy implements LLMProxy {
 
     @Override

chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingRecallParser.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.chat.parser.plugin.embedding;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.parser.LLMProxy;
 import com.tencent.supersonic.chat.parser.PythonLLMProxy;
 import com.tencent.supersonic.chat.parser.plugin.ParseMode;
 import com.tencent.supersonic.chat.parser.plugin.PluginParser;
@@ -27,12 +26,10 @@ import org.springframework.util.CollectionUtils;
 @Slf4j
 public class EmbeddingRecallParser extends PluginParser {
 
-    protected LLMProxy llmInterpreter = ComponentFactory.getLLMProxy();
-
     @Override
     public boolean checkPreCondition(QueryContext queryContext) {
         EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
-        if (StringUtils.isBlank(embeddingConfig.getUrl()) && llmInterpreter instanceof PythonLLMProxy) {
+        if (StringUtils.isBlank(embeddingConfig.getUrl()) && ComponentFactory.getLLMProxy() instanceof PythonLLMProxy) {
             return false;
         }
         List<Plugin> plugins = getPluginList(queryContext);

chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionCallParser.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.chat.parser.plugin.function;
 
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
 import com.tencent.supersonic.chat.parser.PythonLLMProxy;
-import com.tencent.supersonic.chat.parser.LLMProxy;
 import com.tencent.supersonic.chat.parser.plugin.ParseMode;
 import com.tencent.supersonic.chat.parser.plugin.PluginParser;
 import com.tencent.supersonic.chat.plugin.Plugin;
@@ -27,13 +26,11 @@ import org.springframework.util.CollectionUtils;
 @Slf4j
 public class FunctionCallParser extends PluginParser {
 
-    protected LLMProxy llmInterpreter = ComponentFactory.getLLMProxy();
-
     @Override
     public boolean checkPreCondition(QueryContext queryContext) {
         FunctionCallConfig functionCallConfig = ContextUtils.getBean(FunctionCallConfig.class);
         String functionUrl = functionCallConfig.getUrl();
-        if (StringUtils.isBlank(functionUrl) && llmInterpreter instanceof PythonLLMProxy) {
+        if (StringUtils.isBlank(functionUrl) && ComponentFactory.getLLMProxy() instanceof PythonLLMProxy) {
             log.info("functionUrl:{}, skip function parser, queryText:{}", functionUrl,
                     queryContext.getRequest().getQueryText());
             return false;
@@ -84,7 +81,7 @@ public class FunctionCallParser extends PluginParser {
             FunctionReq functionReq = FunctionReq.builder()
                     .queryText(queryContext.getRequest().getQueryText())
                     .pluginConfigs(pluginToFunctionCall).build();
-            functionResp = llmInterpreter.requestFunction(functionReq);
+            functionResp = ComponentFactory.getLLMProxy().requestFunction(functionReq);
         }
         return functionResp;
     }

chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/LLMRequestService.java

@@ -12,7 +12,6 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.config.LLMParserConfig;
 import com.tencent.supersonic.chat.config.OptimizationConfig;
-import com.tencent.supersonic.chat.parser.LLMProxy;
 import com.tencent.supersonic.chat.parser.SatisfactionChecker;
 import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
 import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
@@ -26,13 +25,6 @@ import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.knowledge.service.SchemaService;
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -42,13 +34,17 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
 
 @Slf4j
 @Service
 public class LLMRequestService {
 
-    protected LLMProxy llmProxy = ComponentFactory.getLLMProxy();
-
     protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
     @Autowired
     private LLMParserConfig llmParserConfig;
@@ -60,7 +56,7 @@ public class LLMRequestService {
     private OptimizationConfig optimizationConfig;
 
     public boolean isSkip(QueryContext queryCtx) {
-        if (llmProxy.isSkip(queryCtx)) {
+        if (ComponentFactory.getLLMProxy().isSkip(queryCtx)) {
             return true;
         }
         if (SatisfactionChecker.isSkip(queryCtx)) {
@@ -140,7 +136,7 @@ public class LLMRequestService {
     }
 
     public LLMResp requestLLM(LLMReq llmReq, String modelClusterKey) {
-        return llmProxy.query2sql(llmReq, modelClusterKey);
+        return ComponentFactory.getLLMProxy().query2sql(llmReq, modelClusterKey);
     }
 
     protected List<String> getFieldNameList(QueryContext queryCtx, ModelCluster modelCluster,

chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java

@@ -4,16 +4,22 @@ import com.tencent.supersonic.chat.api.component.SchemaMapper;
 import com.tencent.supersonic.chat.api.component.SemanticCorrector;
 import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
 import com.tencent.supersonic.chat.api.component.SemanticParser;
+import com.tencent.supersonic.chat.parser.JavaLLMProxy;
 import com.tencent.supersonic.chat.parser.LLMProxy;
 import com.tencent.supersonic.chat.parser.sql.llm.ModelResolver;
 import com.tencent.supersonic.chat.processor.ParseResultProcessor;
 import com.tencent.supersonic.chat.query.QueryResponder;
+import com.tencent.supersonic.common.util.ContextUtils;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.core.io.support.SpringFactoriesLoader;
 
+@Slf4j
 public class ComponentFactory {
 
     private static List<SchemaMapper> schemaMappers = new ArrayList<>();
@@ -57,9 +63,21 @@ public class ComponentFactory {
     }
 
     public static LLMProxy getLLMProxy() {
-        if (Objects.isNull(llmProxy)) {
-            llmProxy = init(LLMProxy.class);
+        //1.Preferentially retrieve from environment variables
+        String llmProxyEnv = System.getenv("llmProxy");
+        if (StringUtils.isNotBlank(llmProxyEnv)) {
+            Map<String, LLMProxy> implementations = ContextUtils.getBeansOfType(LLMProxy.class);
+            llmProxy = implementations.entrySet().stream()
+                    .filter(entry -> entry.getKey().equalsIgnoreCase(llmProxyEnv))
+                    .map(Map.Entry::getValue)
+                    .findFirst()
+                    .orElse(null);
         }
+        //2.default JavaLLMProxy
+        if (Objects.isNull(llmProxy)) {
+            llmProxy = ContextUtils.getBean(JavaLLMProxy.class);
+        }
+        log.info("llmProxy:{}", llmProxy);
         return llmProxy;
     }
 

common/src/main/java/com/tencent/supersonic/common/util/ContextUtils.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.common.util;
 
+import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeansException;
 import org.springframework.context.ApplicationContext;
@@ -76,6 +77,10 @@ public class ContextUtils implements ApplicationContextAware {
         return context;
     }
 
+    public static <T> Map<String, T> getBeansOfType(Class<T> requiredType) {
+        return context.getBeansOfType(requiredType);
+    }
+
     public static String getActiveProfile() {
         String activeProfile = context.getEnvironment().getActiveProfiles()[0];
         if (StringUtils.isEmpty(activeProfile)) {

launchers/chat/src/main/resources/META-INF/spring.factories

@@ -18,9 +18,6 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
     com.tencent.supersonic.chat.corrector.GroupByCorrector, \
     com.tencent.supersonic.chat.corrector.HavingCorrector
 
-com.tencent.supersonic.chat.parser.LLMProxy=\
-  com.tencent.supersonic.chat.parser.PythonLLMProxy
-
 com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
     com.tencent.supersonic.knowledge.semantic.RemoteSemanticInterpreter
 

launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java

@@ -1,10 +1,9 @@
 package com.tencent.supersonic;
 
 import com.tencent.supersonic.chat.config.OptimizationConfig;
+import com.tencent.supersonic.chat.parser.JavaLLMProxy;
 import com.tencent.supersonic.chat.parser.sql.llm.SqlExample;
 import com.tencent.supersonic.chat.parser.sql.llm.SqlExampleLoader;
-import com.tencent.supersonic.chat.parser.JavaLLMProxy;
-import com.tencent.supersonic.chat.parser.LLMProxy;
 import com.tencent.supersonic.chat.utils.ComponentFactory;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
@@ -18,7 +17,6 @@ import org.springframework.stereotype.Component;
 @Order(4)
 public class EmbeddingInitListener implements CommandLineRunner {
 
-    protected LLMProxy llmProxy = ComponentFactory.getLLMProxy();
     @Autowired
     private SqlExampleLoader sqlExampleLoader;
     @Autowired
@@ -31,7 +29,7 @@ public class EmbeddingInitListener implements CommandLineRunner {
 
     public void initSqlExamples() {
         try {
-            if (llmProxy instanceof JavaLLMProxy) {
+            if (ComponentFactory.getLLMProxy() instanceof JavaLLMProxy) {
                 List<SqlExample> sqlExamples = sqlExampleLoader.getSqlExamples();
                 String collectionName = optimizationConfig.getText2sqlCollectionName();
                 sqlExampleLoader.addEmbeddingStore(sqlExamples, collectionName);

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -29,9 +29,6 @@ com.tencent.supersonic.chat.processor.ParseResultProcessor=\
     com.tencent.supersonic.chat.processor.TimeCostProcessor, \
     com.tencent.supersonic.chat.processor.RespBuildProcessor
 
-com.tencent.supersonic.chat.parser.LLMProxy=\
-  com.tencent.supersonic.chat.parser.JavaLLMProxy
-
 com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
     com.tencent.supersonic.knowledge.semantic.LocalSemanticInterpreter