Inroduce Loguru for Python logging (#151)

* add loguru to pip requirements

* init logger

* use logger

chat/core/src/main/python/requirements.txt

@@ -4,4 +4,5 @@ fastapi==0.95.1
 chromadb==0.3.21
 tiktoken==0.3.3
 uvicorn[standard]==0.21.1
-pandas==1.5.3
+pandas==1.5.3
+loguru==0.7.2

chat/core/src/main/python/run_config.py

@@ -24,8 +24,8 @@ CHROMA_DB_PERSIST_PATH = os.path.join(PROJECT_DIR_PATH, CHROMA_DB_PERSIST_DIR)
 HF_TEXT2VEC_MODEL_NAME = "GanymedeNil/text2vec-large-chinese"
 
 if __name__ == "__main__":
-    print("PROJECT_DIR_PATH: ", PROJECT_DIR_PATH)
+    logger.info("PROJECT_DIR_PATH: {}", PROJECT_DIR_PATH)
-    print("EMB_MODEL_PATH: ", HF_TEXT2VEC_MODEL_NAME)
+    logger.info("EMB_MODEL_PATH: {}", HF_TEXT2VEC_MODEL_NAME)
-    print("CHROMA_DB_PERSIST_PATH: ", CHROMA_DB_PERSIST_PATH)
+    logger.info("CHROMA_DB_PERSIST_PATH: {}", CHROMA_DB_PERSIST_PATH)
-    print("LLMPARSER_HOST: ", LLMPARSER_HOST)
+    logger.info("LLMPARSER_HOST: {}", LLMPARSER_HOST)
-    print("LLMPARSER_PORT: ", LLMPARSER_PORT)
+    logger.info("LLMPARSER_PORT: {}", LLMPARSER_PORT)

chat/core/src/main/python/services/plugin_call/prompt_construct.py

@@ -5,6 +5,8 @@ import re
 import sys
 from typing import Any, List, Mapping, Union
 
+from loguru import logger
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
@@ -52,10 +54,10 @@ def plugin_selection_output_parse(llm_output: str) -> Union[Mapping[str, str], N
         find_result = re.findall(pattern, llm_output)
         result = find_result[0].strip()
 
-        print("result: ", result)
+        logger.info("result: {}", result)
 
         result_dict = json.loads(result)
-        print("result_dict: ", result_dict)
+        logger.info("result_dict: {}", result_dict)
 
         key_mapping = {"分析过程": "analysis", "选择工具": "toolSelection"}
 
@@ -66,7 +68,7 @@ def plugin_selection_output_parse(llm_output: str) -> Union[Mapping[str, str], N
         }
 
     except Exception as e:
-        print(e)
+        logger.exception(e)
         converted_result_dict = None
 
     return converted_result_dict

chat/core/src/main/python/services/preset_retrieval/run.py

@@ -30,7 +30,7 @@ collection = client.get_or_create_collection(
     metadata={"hnsw:space": "cosine"},
 )  # Get a collection object from an existing collection, by name. If it doesn't exist, create it.
 
-print("init_preset_query_collection_size: ", preset_query_collection_size(collection))
+logger.info("init_preset_query_collection_size: {}", preset_query_collection_size(collection))
 
 
 def preset_query_retrieval_run(
@@ -45,6 +45,6 @@ def preset_query_retrieval_run(
         query_texts_list, parsed_retrieval_res
     )
 
-    print("parsed_retrieval_res_format: ", parsed_retrieval_res_format)
+    logger.info("parsed_retrieval_res_format: {}", parsed_retrieval_res_format)
 
     return parsed_retrieval_res_format

chat/core/src/main/python/services/query_retrieval/run.py

@@ -5,6 +5,8 @@ import sys
 import uuid
 from typing import Any, List, Mapping, Optional, Union
 
+from loguru import logger
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
@@ -28,14 +30,14 @@ solved_query_collection = client.get_or_create_collection(name=SOLVED_QUERY_COLL
                                             embedding_function=emb_func,
                                             metadata={"hnsw:space": "cosine"}
                                             ) # Get a collection object from an existing collection, by name. If it doesn't exist, create it.
-print("init_solved_query_collection_size: ", get_chroma_collection_size(solved_query_collection))
+logger.info("init_solved_query_collection_size: {}", get_chroma_collection_size(solved_query_collection))
 
 
 preset_query_collection = client.get_or_create_collection(name=PRESET_QUERY_COLLECTION_NAME,
                                             embedding_function=emb_func,
                                             metadata={"hnsw:space": "cosine"}
                                             )
-print("init_preset_query_collection_size: ", get_chroma_collection_size(preset_query_collection))
+logger.info("init_preset_query_collection_size: {}", get_chroma_collection_size(preset_query_collection))
 
 class ChromaCollectionRetriever(object):
     def __init__(self, collection:Collection):
@@ -50,7 +52,7 @@ class ChromaCollectionRetriever(object):
         parsed_retrieval_res = parse_retrieval_chroma_collection_query(retrieval_res)
         parsed_retrieval_res_format = chroma_collection_query_retrieval_format(query_texts_list, parsed_retrieval_res)
 
-        print('parsed_retrieval_res_format: ', parsed_retrieval_res_format)
+        logger.info('parsed_retrieval_res_format: {}', parsed_retrieval_res_format)
 
         return parsed_retrieval_res_format
 

chat/core/src/main/python/services/sql/constructor.py

@@ -3,6 +3,8 @@ import os
 import sys
 from typing import List, Mapping
 
+from loguru import logger
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
@@ -21,11 +23,11 @@ def reload_sql_example_collection(
     sql_example_selector: SemanticSimilarityExampleSelector,
     example_nums: int,
 ):
-    print("original sql_examples_collection size:", vectorstore._collection.count())
+    logger.info("original sql_examples_collection size: {}", vectorstore._collection.count())
     new_collection = empty_chroma_collection_2(collection=vectorstore._collection)
     vectorstore._collection = new_collection
 
-    print("emptied sql_examples_collection size:", vectorstore._collection.count())
+    logger.info("emptied sql_examples_collection size: {}", vectorstore._collection.count())
 
     sql_example_selector = SemanticSimilarityExampleSelector(
         vectorstore=sql_examples_vectorstore,
@@ -46,7 +48,7 @@ def reload_sql_example_collection(
     for example in sql_examplars:
         sql_example_selector.add_example(example)
 
-    print("reloaded sql_examples_collection size:", vectorstore._collection.count())
+    logger.info("reloaded sql_examples_collection size: {}", vectorstore._collection.count())
 
     return vectorstore, sql_example_selector
 
@@ -76,11 +78,11 @@ sql_example_selector = SemanticSimilarityExampleSelector(
 )
 
 if sql_examples_vectorstore._collection.count() > 0:
-    print("examples already in sql_vectorstore")
+    logger.info("examples already in sql_vectorstore")
-    print("init sql_vectorstore size:", sql_examples_vectorstore._collection.count())
+    logger.info("init sql_vectorstore size: {}", sql_examples_vectorstore._collection.count())
 
-print("sql_examplars size:", len(sql_examplars))
+logger.info("sql_examplars size: {}", len(sql_examplars))
 sql_examples_vectorstore, sql_example_selector = reload_sql_example_collection(
     sql_examples_vectorstore, sql_examplars, sql_example_selector, example_nums
 )
-print("added sql_vectorstore size:", sql_examples_vectorstore._collection.count())
+logger.info("added sql_vectorstore size: {}", sql_examples_vectorstore._collection.count())

chat/core/src/main/python/services/sql/examples_reload_run.py

@@ -23,7 +23,7 @@ def text2dsl_setting_update(
 ):
 
     url = f"http://{llm_parser_host}:{llm_parser_port}/query2sql_setting_update/"
-    print("url: ", url)
+    logger.info("url: {}", url)
     payload = {
         "sqlExamplars": sql_examplars,
         "exampleNums": example_nums,
@@ -31,7 +31,7 @@ def text2dsl_setting_update(
     }
     headers = {"content-type": "application/json"}
     response = requests.post(url, data=json.dumps(payload), headers=headers)
-    print(response.text)
+    logger.info(response.text)
 
 
 if __name__ == "__main__":

chat/core/src/main/python/services/sql/output_parser.py

@@ -10,7 +10,7 @@ def schema_link_parse(schema_link_output):
             0
         ].strip()
     except Exception as e:
-        print(e)
+        logger.exception(e)
         schema_link_output = None
 
     return schema_link_output
@@ -27,7 +27,7 @@ def combo_schema_link_parse(schema_linking_sql_combo_output: str):
         else:
             schema_links = None
     except Exception as e:
-        print(e)
+        logger.info(e)
         schema_links = None
 
     return schema_links
@@ -44,7 +44,7 @@ def combo_sql_parse(schema_linking_sql_combo_output: str):
         else:
             sql = None
     except Exception as e:
-        print(e)
+        logger.exception(e)
         sql = None
 
     return sql

chat/core/src/main/python/services/sql/run.py

@@ -2,6 +2,8 @@ import os
 import sys
 from typing import List, Union, Mapping
 
+from loguru import logger
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
@@ -63,10 +65,10 @@ class Text2DSLAgent(object):
         linking: Union[List[Mapping[str, str]], None] = None,
     ):
 
-        print("query_text: ", query_text)
+        logger.info("query_text: {}", query_text)
-        print("schema: ", schema)
+        logger.info("schema: {}", schema)
-        print("current_date: ", current_date)
+        logger.info("current_date: {}", current_date)
-        print("prior_schema_links: ", linking)
+        logger.info("prior_schema_links: {}", linking)
 
         if linking is not None:
             prior_schema_links = {
@@ -85,7 +87,7 @@ class Text2DSLAgent(object):
             prior_schema_links,
             self.sql_example_selector,
         )
-        print("schema_linking_prompt->", schema_linking_prompt)
+        logger.info("schema_linking_prompt-> {}", schema_linking_prompt)
         schema_link_output = self.llm(schema_linking_prompt)
         schema_link_str = self.schema_link_parse(schema_link_output)
 
@@ -96,7 +98,7 @@ class Text2DSLAgent(object):
             current_date,
             self.sql_example_selector,
         )
-        print("sql_prompt->", sql_prompt)
+        logger.info("sql_prompt->", sql_prompt)
         sql_output = self.llm(sql_prompt)
 
         resp = dict()
@@ -111,7 +113,7 @@ class Text2DSLAgent(object):
 
         resp["sqlOutput"] = sql_output
 
-        print("resp: ", resp)
+        logger.info("resp: ", resp)
 
         return resp
 
@@ -123,10 +125,10 @@ class Text2DSLAgent(object):
         linking: Union[List[Mapping[str, str]], None] = None,
     ):
 
-        print("query_text: ", query_text)
+        logger.info("query_text: ", query_text)
-        print("schema: ", schema)
+        logger.info("schema: ", schema)
-        print("current_date: ", current_date)
+        logger.info("current_date: ", current_date)
-        print("prior_schema_links: ", linking)
+        logger.info("prior_schema_links: ", linking)
 
         if linking is not None:
             prior_schema_links = {
@@ -146,7 +148,7 @@ class Text2DSLAgent(object):
             prior_schema_links,
             self.sql_example_selector,
         )
-        print("schema_linking_sql_combo_prompt->", schema_linking_sql_combo_prompt)
+        logger.info("schema_linking_sql_combo_prompt->", schema_linking_sql_combo_prompt)
         schema_linking_sql_combo_output = self.llm(schema_linking_sql_combo_prompt)
 
         schema_linking_str = self.combo_schema_link_parse(
@@ -165,7 +167,7 @@ class Text2DSLAgent(object):
         resp["schemaLinkStr"] = schema_linking_str
         resp["sqlOutput"] = sql_str
 
-        print("resp: ", resp)
+        logger.info("resp: ", resp)
 
         return resp
 

chat/core/src/main/python/supersonic_llmparser.py

@@ -4,6 +4,10 @@ import sys
 
 import uvicorn
 
+from util.logging_utils import init_logger
+
+init_logger()
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 

chat/core/src/main/python/util/chromadb_instance.py

@@ -4,6 +4,7 @@ from typing import Any, List, Mapping, Optional, Union
 import chromadb
 from chromadb.api import Collection
 from chromadb.config import Settings
+from loguru import logger
 
 from run_config import CHROMA_DB_PERSIST_PATH
 
@@ -29,7 +30,7 @@ def empty_chroma_collection_2(collection:Collection):
 
     size_of_new_collection = new_collection.count()
 
-    print(f'Collection {collection_name} emptied. Size of new collection: {size_of_new_collection}')
+    logger.info(f'Collection {collection_name} emptied. Size of new collection: {size_of_new_collection}')
 
     return new_collection
 
@@ -74,7 +75,7 @@ def query_chroma_collection(collection:Collection, query_texts:List[str],
     else:
         outer_filter = None
 
-    print('outer_filter: ', outer_filter)
+    logger.info('outer_filter: ', outer_filter)
     res = collection.query(query_texts=query_texts, n_results=n_results, where=outer_filter)
     return res
 

chat/core/src/main/python/util/logging_utils.py

@@ -0,0 +1,6 @@
+from loguru import logger
+
+
+def init_logger():
+    logger.remove()
+    logger.add("llmparser.info.log")