(improvement)(Chat) Move python module from Chat To Headless (#823)

Co-authored-by: jolunoluo

headless/python/instances/chromadb_instance.py

@@ -0,0 +1,21 @@
+# -*- coding:utf-8 -*-
+from typing import Any, List, Mapping, Optional, Union
+
+import chromadb
+from chromadb.api import Collection
+from chromadb.config import Settings
+
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from config.config_parse import CHROMA_DB_PERSIST_PATH
+
+
+client = chromadb.Client(
+    Settings(
+        chroma_db_impl="duckdb+parquet",
+        persist_directory=CHROMA_DB_PERSIST_PATH,  # Optional, defaults to .chromadb/ in the current directory
+    )
+)

headless/python/instances/llm_instance.py

@@ -0,0 +1,21 @@
+# -*- coding:utf-8 -*-
+from langchain import llms
+
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from config.config_parse import LLM_PROVIDER_NAME, llm_config_dict
+
+
+def get_llm_provider(llm_provider_name: str, llm_config_dict: dict):
+    if llm_provider_name in llms.type_to_cls_dict:
+        llm_provider = llms.type_to_cls_dict[llm_provider_name]
+        llm = llm_provider(**llm_config_dict)
+        return llm
+    else:
+        raise Exception("llm_provider_name is not supported: {}".format(llm_provider_name))
+
+
+llm = get_llm_provider(LLM_PROVIDER_NAME, llm_config_dict)

headless/python/instances/logging_instance.py

@@ -0,0 +1,14 @@
+# -*- coding:utf-8 -*-
+
+from loguru import logger
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from config.config_parse import LOG_FILE_PATH
+
+logger.remove() #remove the old handler. Else, the old one will work along with the new one you've added below'
+logger.add(LOG_FILE_PATH, rotation="500 MB", retention="7 days", format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}", level="INFO")

headless/python/instances/text2vec_instance.py

@@ -0,0 +1,11 @@
+# -*- coding:utf-8 -*-
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from utils.text2vec import Text2VecEmbeddingFunction
+
+emb_func = Text2VecEmbeddingFunction()