From 1e1803d148c7848b4fc43a0246cd072de66ddd90 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Sat, 7 Oct 2023 10:42:02 +0800 Subject: [PATCH] Inroduce Loguru for Python logging (#151) * add loguru to pip requirements * init logger * use logger --- chat/core/src/main/python/requirements.txt | 3 +- chat/core/src/main/python/run_config.py | 10 +++---- .../services/plugin_call/prompt_construct.py | 8 ++++-- .../python/services/preset_retrieval/run.py | 4 +-- .../python/services/query_retrieval/run.py | 8 ++++-- .../main/python/services/sql/constructor.py | 16 ++++++----- .../services/sql/examples_reload_run.py | 4 +-- .../main/python/services/sql/output_parser.py | 6 ++-- chat/core/src/main/python/services/sql/run.py | 28 ++++++++++--------- .../src/main/python/supersonic_llmparser.py | 4 +++ .../src/main/python/util/chromadb_instance.py | 5 ++-- .../src/main/python/util/logging_utils.py | 6 ++++ 12 files changed, 61 insertions(+), 41 deletions(-) create mode 100644 chat/core/src/main/python/util/logging_utils.py diff --git a/chat/core/src/main/python/requirements.txt b/chat/core/src/main/python/requirements.txt index a5a48d046..5bedde364 100644 --- a/chat/core/src/main/python/requirements.txt +++ b/chat/core/src/main/python/requirements.txt @@ -4,4 +4,5 @@ fastapi==0.95.1 chromadb==0.3.21 tiktoken==0.3.3 uvicorn[standard]==0.21.1 -pandas==1.5.3 \ No newline at end of file +pandas==1.5.3 +loguru==0.7.2 diff --git a/chat/core/src/main/python/run_config.py b/chat/core/src/main/python/run_config.py index b8a51176e..8be2a98c7 100644 --- a/chat/core/src/main/python/run_config.py +++ b/chat/core/src/main/python/run_config.py @@ -24,8 +24,8 @@ CHROMA_DB_PERSIST_PATH = os.path.join(PROJECT_DIR_PATH, CHROMA_DB_PERSIST_DIR) HF_TEXT2VEC_MODEL_NAME = "GanymedeNil/text2vec-large-chinese" if __name__ == "__main__": - print("PROJECT_DIR_PATH: ", PROJECT_DIR_PATH) - print("EMB_MODEL_PATH: ", HF_TEXT2VEC_MODEL_NAME) - print("CHROMA_DB_PERSIST_PATH: ", CHROMA_DB_PERSIST_PATH) - print("LLMPARSER_HOST: ", LLMPARSER_HOST) - print("LLMPARSER_PORT: ", LLMPARSER_PORT) + logger.info("PROJECT_DIR_PATH: {}", PROJECT_DIR_PATH) + logger.info("EMB_MODEL_PATH: {}", HF_TEXT2VEC_MODEL_NAME) + logger.info("CHROMA_DB_PERSIST_PATH: {}", CHROMA_DB_PERSIST_PATH) + logger.info("LLMPARSER_HOST: {}", LLMPARSER_HOST) + logger.info("LLMPARSER_PORT: {}", LLMPARSER_PORT) diff --git a/chat/core/src/main/python/services/plugin_call/prompt_construct.py b/chat/core/src/main/python/services/plugin_call/prompt_construct.py index 00376b9da..a220acf5a 100644 --- a/chat/core/src/main/python/services/plugin_call/prompt_construct.py +++ b/chat/core/src/main/python/services/plugin_call/prompt_construct.py @@ -5,6 +5,8 @@ import re import sys from typing import Any, List, Mapping, Union +from loguru import logger + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -52,10 +54,10 @@ def plugin_selection_output_parse(llm_output: str) -> Union[Mapping[str, str], N find_result = re.findall(pattern, llm_output) result = find_result[0].strip() - print("result: ", result) + logger.info("result: {}", result) result_dict = json.loads(result) - print("result_dict: ", result_dict) + logger.info("result_dict: {}", result_dict) key_mapping = {"分析过程": "analysis", "选择工具": "toolSelection"} @@ -66,7 +68,7 @@ def plugin_selection_output_parse(llm_output: str) -> Union[Mapping[str, str], N } except Exception as e: - print(e) + logger.exception(e) converted_result_dict = None return converted_result_dict diff --git a/chat/core/src/main/python/services/preset_retrieval/run.py b/chat/core/src/main/python/services/preset_retrieval/run.py index 87fe4fa91..a2df902b6 100644 --- a/chat/core/src/main/python/services/preset_retrieval/run.py +++ b/chat/core/src/main/python/services/preset_retrieval/run.py @@ -30,7 +30,7 @@ collection = client.get_or_create_collection( metadata={"hnsw:space": "cosine"}, ) # Get a collection object from an existing collection, by name. If it doesn't exist, create it. -print("init_preset_query_collection_size: ", preset_query_collection_size(collection)) +logger.info("init_preset_query_collection_size: {}", preset_query_collection_size(collection)) def preset_query_retrieval_run( @@ -45,6 +45,6 @@ def preset_query_retrieval_run( query_texts_list, parsed_retrieval_res ) - print("parsed_retrieval_res_format: ", parsed_retrieval_res_format) + logger.info("parsed_retrieval_res_format: {}", parsed_retrieval_res_format) return parsed_retrieval_res_format diff --git a/chat/core/src/main/python/services/query_retrieval/run.py b/chat/core/src/main/python/services/query_retrieval/run.py index 0a0c82107..93f135a0b 100644 --- a/chat/core/src/main/python/services/query_retrieval/run.py +++ b/chat/core/src/main/python/services/query_retrieval/run.py @@ -5,6 +5,8 @@ import sys import uuid from typing import Any, List, Mapping, Optional, Union +from loguru import logger + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -28,14 +30,14 @@ solved_query_collection = client.get_or_create_collection(name=SOLVED_QUERY_COLL embedding_function=emb_func, metadata={"hnsw:space": "cosine"} ) # Get a collection object from an existing collection, by name. If it doesn't exist, create it. -print("init_solved_query_collection_size: ", get_chroma_collection_size(solved_query_collection)) +logger.info("init_solved_query_collection_size: {}", get_chroma_collection_size(solved_query_collection)) preset_query_collection = client.get_or_create_collection(name=PRESET_QUERY_COLLECTION_NAME, embedding_function=emb_func, metadata={"hnsw:space": "cosine"} ) -print("init_preset_query_collection_size: ", get_chroma_collection_size(preset_query_collection)) +logger.info("init_preset_query_collection_size: {}", get_chroma_collection_size(preset_query_collection)) class ChromaCollectionRetriever(object): def __init__(self, collection:Collection): @@ -50,7 +52,7 @@ class ChromaCollectionRetriever(object): parsed_retrieval_res = parse_retrieval_chroma_collection_query(retrieval_res) parsed_retrieval_res_format = chroma_collection_query_retrieval_format(query_texts_list, parsed_retrieval_res) - print('parsed_retrieval_res_format: ', parsed_retrieval_res_format) + logger.info('parsed_retrieval_res_format: {}', parsed_retrieval_res_format) return parsed_retrieval_res_format diff --git a/chat/core/src/main/python/services/sql/constructor.py b/chat/core/src/main/python/services/sql/constructor.py index d2dcc1bc4..aed344835 100644 --- a/chat/core/src/main/python/services/sql/constructor.py +++ b/chat/core/src/main/python/services/sql/constructor.py @@ -3,6 +3,8 @@ import os import sys from typing import List, Mapping +from loguru import logger + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -21,11 +23,11 @@ def reload_sql_example_collection( sql_example_selector: SemanticSimilarityExampleSelector, example_nums: int, ): - print("original sql_examples_collection size:", vectorstore._collection.count()) + logger.info("original sql_examples_collection size: {}", vectorstore._collection.count()) new_collection = empty_chroma_collection_2(collection=vectorstore._collection) vectorstore._collection = new_collection - print("emptied sql_examples_collection size:", vectorstore._collection.count()) + logger.info("emptied sql_examples_collection size: {}", vectorstore._collection.count()) sql_example_selector = SemanticSimilarityExampleSelector( vectorstore=sql_examples_vectorstore, @@ -46,7 +48,7 @@ def reload_sql_example_collection( for example in sql_examplars: sql_example_selector.add_example(example) - print("reloaded sql_examples_collection size:", vectorstore._collection.count()) + logger.info("reloaded sql_examples_collection size: {}", vectorstore._collection.count()) return vectorstore, sql_example_selector @@ -76,11 +78,11 @@ sql_example_selector = SemanticSimilarityExampleSelector( ) if sql_examples_vectorstore._collection.count() > 0: - print("examples already in sql_vectorstore") - print("init sql_vectorstore size:", sql_examples_vectorstore._collection.count()) + logger.info("examples already in sql_vectorstore") + logger.info("init sql_vectorstore size: {}", sql_examples_vectorstore._collection.count()) -print("sql_examplars size:", len(sql_examplars)) +logger.info("sql_examplars size: {}", len(sql_examplars)) sql_examples_vectorstore, sql_example_selector = reload_sql_example_collection( sql_examples_vectorstore, sql_examplars, sql_example_selector, example_nums ) -print("added sql_vectorstore size:", sql_examples_vectorstore._collection.count()) +logger.info("added sql_vectorstore size: {}", sql_examples_vectorstore._collection.count()) diff --git a/chat/core/src/main/python/services/sql/examples_reload_run.py b/chat/core/src/main/python/services/sql/examples_reload_run.py index c2b614614..ea374932a 100644 --- a/chat/core/src/main/python/services/sql/examples_reload_run.py +++ b/chat/core/src/main/python/services/sql/examples_reload_run.py @@ -23,7 +23,7 @@ def text2dsl_setting_update( ): url = f"http://{llm_parser_host}:{llm_parser_port}/query2sql_setting_update/" - print("url: ", url) + logger.info("url: {}", url) payload = { "sqlExamplars": sql_examplars, "exampleNums": example_nums, @@ -31,7 +31,7 @@ def text2dsl_setting_update( } headers = {"content-type": "application/json"} response = requests.post(url, data=json.dumps(payload), headers=headers) - print(response.text) + logger.info(response.text) if __name__ == "__main__": diff --git a/chat/core/src/main/python/services/sql/output_parser.py b/chat/core/src/main/python/services/sql/output_parser.py index eeeb6bce0..210c2e1f3 100644 --- a/chat/core/src/main/python/services/sql/output_parser.py +++ b/chat/core/src/main/python/services/sql/output_parser.py @@ -10,7 +10,7 @@ def schema_link_parse(schema_link_output): 0 ].strip() except Exception as e: - print(e) + logger.exception(e) schema_link_output = None return schema_link_output @@ -27,7 +27,7 @@ def combo_schema_link_parse(schema_linking_sql_combo_output: str): else: schema_links = None except Exception as e: - print(e) + logger.info(e) schema_links = None return schema_links @@ -44,7 +44,7 @@ def combo_sql_parse(schema_linking_sql_combo_output: str): else: sql = None except Exception as e: - print(e) + logger.exception(e) sql = None return sql diff --git a/chat/core/src/main/python/services/sql/run.py b/chat/core/src/main/python/services/sql/run.py index 82ca3058a..2dbad25bb 100644 --- a/chat/core/src/main/python/services/sql/run.py +++ b/chat/core/src/main/python/services/sql/run.py @@ -2,6 +2,8 @@ import os import sys from typing import List, Union, Mapping +from loguru import logger + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -63,10 +65,10 @@ class Text2DSLAgent(object): linking: Union[List[Mapping[str, str]], None] = None, ): - print("query_text: ", query_text) - print("schema: ", schema) - print("current_date: ", current_date) - print("prior_schema_links: ", linking) + logger.info("query_text: {}", query_text) + logger.info("schema: {}", schema) + logger.info("current_date: {}", current_date) + logger.info("prior_schema_links: {}", linking) if linking is not None: prior_schema_links = { @@ -85,7 +87,7 @@ class Text2DSLAgent(object): prior_schema_links, self.sql_example_selector, ) - print("schema_linking_prompt->", schema_linking_prompt) + logger.info("schema_linking_prompt-> {}", schema_linking_prompt) schema_link_output = self.llm(schema_linking_prompt) schema_link_str = self.schema_link_parse(schema_link_output) @@ -96,7 +98,7 @@ class Text2DSLAgent(object): current_date, self.sql_example_selector, ) - print("sql_prompt->", sql_prompt) + logger.info("sql_prompt->", sql_prompt) sql_output = self.llm(sql_prompt) resp = dict() @@ -111,7 +113,7 @@ class Text2DSLAgent(object): resp["sqlOutput"] = sql_output - print("resp: ", resp) + logger.info("resp: ", resp) return resp @@ -123,10 +125,10 @@ class Text2DSLAgent(object): linking: Union[List[Mapping[str, str]], None] = None, ): - print("query_text: ", query_text) - print("schema: ", schema) - print("current_date: ", current_date) - print("prior_schema_links: ", linking) + logger.info("query_text: ", query_text) + logger.info("schema: ", schema) + logger.info("current_date: ", current_date) + logger.info("prior_schema_links: ", linking) if linking is not None: prior_schema_links = { @@ -146,7 +148,7 @@ class Text2DSLAgent(object): prior_schema_links, self.sql_example_selector, ) - print("schema_linking_sql_combo_prompt->", schema_linking_sql_combo_prompt) + logger.info("schema_linking_sql_combo_prompt->", schema_linking_sql_combo_prompt) schema_linking_sql_combo_output = self.llm(schema_linking_sql_combo_prompt) schema_linking_str = self.combo_schema_link_parse( @@ -165,7 +167,7 @@ class Text2DSLAgent(object): resp["schemaLinkStr"] = schema_linking_str resp["sqlOutput"] = sql_str - print("resp: ", resp) + logger.info("resp: ", resp) return resp diff --git a/chat/core/src/main/python/supersonic_llmparser.py b/chat/core/src/main/python/supersonic_llmparser.py index a1c7bc3c3..db7c90a3c 100644 --- a/chat/core/src/main/python/supersonic_llmparser.py +++ b/chat/core/src/main/python/supersonic_llmparser.py @@ -4,6 +4,10 @@ import sys import uvicorn +from util.logging_utils import init_logger + +init_logger() + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) diff --git a/chat/core/src/main/python/util/chromadb_instance.py b/chat/core/src/main/python/util/chromadb_instance.py index 26a4bffc8..c6fcf1953 100644 --- a/chat/core/src/main/python/util/chromadb_instance.py +++ b/chat/core/src/main/python/util/chromadb_instance.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping, Optional, Union import chromadb from chromadb.api import Collection from chromadb.config import Settings +from loguru import logger from run_config import CHROMA_DB_PERSIST_PATH @@ -29,7 +30,7 @@ def empty_chroma_collection_2(collection:Collection): size_of_new_collection = new_collection.count() - print(f'Collection {collection_name} emptied. Size of new collection: {size_of_new_collection}') + logger.info(f'Collection {collection_name} emptied. Size of new collection: {size_of_new_collection}') return new_collection @@ -74,7 +75,7 @@ def query_chroma_collection(collection:Collection, query_texts:List[str], else: outer_filter = None - print('outer_filter: ', outer_filter) + logger.info('outer_filter: ', outer_filter) res = collection.query(query_texts=query_texts, n_results=n_results, where=outer_filter) return res diff --git a/chat/core/src/main/python/util/logging_utils.py b/chat/core/src/main/python/util/logging_utils.py new file mode 100644 index 000000000..8928db9b6 --- /dev/null +++ b/chat/core/src/main/python/util/logging_utils.py @@ -0,0 +1,6 @@ +from loguru import logger + + +def init_logger(): + logger.remove() + logger.add("llmparser.info.log")