(improvement)(chat) fix embedding null pointer (#712)

evaluation/README_CN.md

@@ -1,10 +1,8 @@
 # 评测流程
 
 1. 正常启动项目(必须包括LLM服务)
-2. 将要评测问题放到evalution/data目录下，如：internet.txt；将要评测问题对应的SQL也放到evalution/data目录下，如：gold_example_dusql.txt。
-3. 执行evalution.sh脚本，主要包括构建表数据、获取模型预测结果，执行对比逻辑。可以在命令行看到执行准确率，错误case会写到同目录的eval.json文件中。
+2. 执行evalution.sh脚本，主要包括构建表数据、获取模型预测结果，执行对比逻辑。可以在命令行看到执行准确率，错误case会写到同目录的error_case.json文件中。
 
 # 评测意义
 
-制定大模型评估框架对于提示词或代码更改的影响至关重要，可以帮助我们了解这些变化是否会提高或降低准确率、响应速度。 随着产品规模的扩大，如果没有这样的框架，就会发现自己在盲目地调整黑匣子，有助于帮助我们减少问题、提高效率、增强模型能力。大模型评测的核心目的是确定模型的"聪明"程度，深入探讨其性能、特点和局限性，为行业应用提供方向。
-通过评测，我们可以更好地了解模型的性能、特点、价值、局限性和潜在风险，并为其发展和应用提供支持，具有重要意义。
+制定评估工具对于提示词或代码更改的影响至关重要，方便supersonic快速对接其他模型、更改配置，可以帮助我们了解这些变化是否会提高或降低准确率、响应速度。

evaluation/build_pred_result.py

@@ -48,7 +48,7 @@ def get_pred_result():
     config_file=current_directory+"/config/config.yaml"
     with open(config_file, 'r') as file:
         config = yaml.safe_load(file)
-    input_path=current_directory+"/data/"+config["domain"]+".txt"
+    input_path=current_directory+"/data/"+"internet.txt"
     pred_sql_path = current_directory+"/data/"+"pred_example_dusql.txt"
     pred_sql_exist=os.path.exists(pred_sql_path)
     if pred_sql_exist:

evaluation/build_tables.py

@@ -288,7 +288,7 @@ def build_table():
     with open(config_file, 'r') as file:
         config = yaml.safe_load(file)
     db_path=current_directory+"/data/"
-    db_file=db_path+config["domain"]+".db"
+    db_file=db_path+"internet.db"
     db_exist=os.path.exists(db_file)
     if db_exist:
         os.remove(db_file)
@@ -301,7 +301,7 @@ if __name__ == '__main__':
     with open(config_file, 'r') as file:
         config = yaml.safe_load(file)
     db_path=current_directory+"/data/"
-    db_file=db_path+config["domain"]+".db"
+    db_file=db_path+"internet.db"
     db_exist=os.path.exists(db_file)
     if db_exist:
         os.remove(db_file)
@@ -311,30 +311,3 @@ if __name__ == '__main__':
     #build_china_travel_agency(path)
 
 
-
-
-
-
-
-    # sql="SELECT T3.company_name, T3.annual_turnover, T2.brand_name, T1.revenue_proportion FROM company_revenue AS T1 JOIN brand AS T2 JOIN company AS T3 ON T1.brand_id = T2.brand_id AND T1.company_id = T3.company_id"
-    # query_sql(sql)
-    # sql =" select * from ( SELECT brand_name, company_name, sum(revenue_proportion), sum(annual_turnover) FROM (select `sys_imp_date`, `company_name`, `brand_name`, `annual_turnover`, `revenue_proportion`from(select `sys_imp_date`, `company_name`, `brand_name`, `annual_turnover` as `annual_turnover`, `revenue_proportion` as `revenue_proportion`from(select `annual_turnover` as `annual_turnover`, `revenue_proportion` as `revenue_proportion`, `sys_imp_date`, `company_name`, `brand_name`from(select `src1_brand`.`sys_imp_date` as `sys_imp_date`, `src1_company`.`annual_turnover` as `annual_turnover`, `src1_company_revenue`.`revenue_proportion` as `revenue_proportion`, `src1_brand`.`company_id` as `company_id`, `src1_company`.`company_name` as `company_name`, `src1_brand`.`brand_name` as `brand_name`, `src1_brand`.`brand_id` as `brand_id`from(select `annual_turnover` as `annual_turnover`, `imp_date` as `sys_imp_date`, `company_id`, `company_name` as `company_name`, `imp_date` as `imp_date`from(select `imp_date`, `company_id`, `company_name`, `headquarter_address`, `company_established_time`, `founder`, `ceo`, `annual_turnover`, `employee_count`from`company`) as `company`) as `src1_company`inner join (select `revenue_proportion` as `revenue_proportion`, `imp_date` as `sys_imp_date`, `company_id`, `brand_id`, `imp_date` as `imp_date`from(select `imp_date`, `company_id`, `brand_id`, `revenue_proportion`, `profit_proportion`, `expenditure_proportion`from`company_revenue`) as `company_revenue`) as `src1_company_revenue` on `src1_company`.`company_id` = `src1_company_revenue`.`company_id`inner join (select `imp_date` as `sys_imp_date`, `company_id`, `brand_name` as `brand_name`, `brand_id`, `imp_date` as `imp_date`from(select `imp_date`, `brand_id`, `brand_name`, `brand_established_time`, `company_id`, `legal_representative`, `registered_capital`from`brand`) as `brand`) as `src1_brand` on `src1_company`.`company_id` = `src1_brand`.`company_id`) as `src11_`) as `company_company_revenue_brand_0`) as `company_company_revenue_brand_1`) t_103 WHERE sys_imp_date = '2024-01-11' GROUP BY brand_name, company_name, sys_imp_date ORDER BY sum(revenue_proportion) DESC ) a limit 1000 "
-    # a=query_sql(sql)
-    # print(a[0][0])
-
-
-    # import pymysql
-    #
-    # db = pymysql.connect(host="11.154.212.211", user="semantic", password="semantic2023", database="internet", port=3306)
-    #
-    # # 使用cursor()方法获取操作游标
-    # cursor = db.cursor()
-    #
-    # # 使用execute方法执行SQL语句
-    # cursor.execute("select * from company")
-    #
-    # # 使用 fetchone() 方法获取一条数据
-    # data = cursor.fetchone()
-    # print(data)
-
-

evaluation/config/config.yaml

@@ -1,4 +1,3 @@
 chat_id: 3
 agent_id: 4
-domain: internet
 url: http://localhost:9080

evaluation/error_case.json

@@ -0,0 +1,17 @@
+[
+    {
+        "query": "在各公司所有品牌收入排名中，给出每一个品牌，其所在公司以及收入占该公司的总收入比例，同时给出该公司的年营业额",
+        "gold_sql": "SELECT T3.company_name, T3.annual_turnover, T2.brand_name, T1.revenue_proportion FROM company_revenue AS T1 JOIN brand AS T2 JOIN company AS T3 ON T1.brand_id = T2.brand_id AND T1.company_id = T3.company_id",
+        "pred_sql": "select * from tablea"
+    },
+    {
+        "query": "在各公司所有品牌收入排名中，给出每一个品牌，其所在公司以及收入占该公司的总收入比例",
+        "gold_sql": "SELECT T3.company_name, T2.brand_name, T1.revenue_proportion FROM company_revenue AS T1 JOIN brand AS T2 JOIN company AS T3 ON T1.brand_id = T2.brand_id AND T1.company_id = T3.company_id",
+        "pred_sql": "select * from tablea"
+    },
+    {
+        "query": "在各公司所有品牌收入排名中，给出每一个品牌和其法人，其所在公司以及收入占该公司的总收入比例",
+        "gold_sql": "SELECT T3.company_name, T2.brand_name, T2.legal_representative, T1.revenue_proportion FROM company_revenue AS T1 JOIN brand AS T2 JOIN company AS T3 ON T1.brand_id = T2.brand_id AND T1.company_id = T3.company_id",
+        "pred_sql": "select * from tablea"
+    }
+]

evaluation/evaluation.py

@@ -513,7 +513,6 @@ def evaluate(gold, predict, db_dir, etype, kmaps,query_path):
         db_name = db
         # db = os.path.join(db_dir, db, db + ".sqlite")
         db = os.path.join(db_dir,db + ".db")
-        print(db)
         schema = Schema(get_schema(db))
         g_sql = get_sql(schema, g_str)
         hardness = evaluator.eval_hardness(g_sql)
@@ -597,7 +596,7 @@ def evaluate(gold, predict, db_dir, etype, kmaps,query_path):
     print_scores(scores, etype)
     print(scores['all']['exec'])
     current_directory = os.path.dirname(os.path.abspath(__file__))
-    file_name=current_directory+"/eval.json"
+    file_name=current_directory+"/error_case.json"
     json_exist=os.path.exists(file_name)
     if json_exist:
         os.remove(file_name)
@@ -884,11 +883,11 @@ def get_evaluation_result():
         config = yaml.safe_load(file)
     db_dir=current_directory+"/data"
     db_path=current_directory+"/data/"
-    db_file=db_path+config["domain"]+".db"
+    db_file=db_path+"internet.db"
     pred = current_directory+"/data/"+"pred_example_dusql.txt"
     gold = current_directory+"/data/"+"gold_example_dusql.txt"
     table= current_directory+"/data/"+"tables_dusql.json"
-    query_path=current_directory+"/data/"+config["domain"]+".txt"
+    query_path=current_directory+"/data/"+"internet.txt"
     etype="exec"
     kmaps = build_foreign_key_map_from_json(table)
 
@@ -900,7 +899,7 @@ def remove_unused_file():
     with open(config_file, 'r') as file:
         config = yaml.safe_load(file)
     db_path=current_directory+"/data/"
-    db_file=db_path+config["domain"]+".db"
+    db_file=db_path+"internet.db"
     pred_file = current_directory+"/data/"+"pred_example_dusql.txt"
 
     db_exist=os.path.exists(db_file)