mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 19:51:00 +00:00
add auto-CoT feature (#483)
* 1.refactor the retrieval module. 2.refactor the http service module. 3.upgrade text2sql output format the parse for absolute time related expression in query. * fix bug. * upgrade the config module, now support config llm suppoted by langchain. * fix conflicts. * update text2sql config reload to be compitable with new config format. * modify default config. * 1.add self-consistency feature for text2sql. 2.upgrade llm api call from sync to async. 3.refactor text2sql module. 4. refactor semantical retriever modules. * merege with upstream master * add general retrieve service. * add api service for sql_agent for crud opereations of few-shots examples. * modify requirements * add auto-cot feature --------- Co-authored-by: shaoweigong <shaoweigong@tencent.com>
This commit is contained in:
codescracker
374
chat/core/src/main/python/few_shot_example/sql_examplar.py
Normal file
374
chat/core/src/main/python/few_shot_example/sql_examplar.py
Normal file
@@ -0,0 +1,374 @@
|
||||
examplars= [
|
||||
{ "currentDate":"2020-12-01",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
|
||||
"question":"比较jackjchen和robinlee在内容库的访问次数",
|
||||
"priorSchemaLinks":"""['jackjchen'->用户名, 'robinlee'->用户名]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“比较jackjchen和robinlee在内容库的访问次数“中，我们被问：
|
||||
“比较jackjchen和robinlee”，所以我们需要column=[用户名],cell values = ['jackjchen', 'robinlee'],所以有[用户名:('jackjchen', 'robinlee')]
|
||||
”内容库的访问次数“，所以我们需要column=[访问次数]""",
|
||||
"schemaLinks":"""["用户名":("'jackjchen'", "'robinlee'"), "访问次数"]""",
|
||||
"sql":"""select 用户名, 访问次数 from 内容库产品 where 用户名 in ('jackjchen', 'robinlee')"""
|
||||
},
|
||||
{ "currentDate":"2022-11-06",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
|
||||
"question":"内容库近12个月访问人数 按部门",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库近12个月访问人数 按部门“中，我们被问：
|
||||
”内容库近12个月“，所以我们需要column=[数据日期],cell values = [12],所以有[数据日期:(12)]
|
||||
“访问人数”，所以我们需要column=[访问人数]
|
||||
”按部门“，所以我们需要column=[部门]""",
|
||||
"schemaLinks":"""["数据日期":(12), "访问人数", "部门"]""",
|
||||
"sql":"""select 部门, 数据日期, 访问人数 from 内容库产品 where datediff('month', 数据日期, '2022-11-06') <= 12 """
|
||||
},
|
||||
{ "currentDate":"2023-04-21",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
|
||||
"question":"内容库美术部、技术研发部的访问时长",
|
||||
"priorSchemaLinks":"""['美术部'->部门, '技术研发部'->部门]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库美术部、技术研发部的访问时长“中，我们被问：
|
||||
“访问时长”，所以我们需要column=[访问时长]
|
||||
”内容库美术部、技术研发部“，所以我们需要column=[部门], cell values = ['美术部', '技术研发部'],所以有[部门:('美术部', '技术研发部')]""",
|
||||
"schemaLinks":"""["访问时长", "部门":("'美术部'", "'技术研发部'")]""",
|
||||
"sql":"""select 部门, 访问时长 from 内容库产品 where 部门 in ('美术部', '技术研发部')"""
|
||||
},
|
||||
{ "currentDate":"2023-08-21",
|
||||
"tableName":"严选",
|
||||
"fieldsList":"""["严选版权归属系", "付费模式", "结算播放份额", "付费用户结算播放份额", "数据日期"]""",
|
||||
"question":"近3天海田飞系MPPM结算播放份额",
|
||||
"priorSchemaLinks":"""['海田飞系'->严选版权归属系]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“近3天海田飞系MPPM结算播放份额“中，我们被问：
|
||||
“MPPM结算播放份额”，所以我们需要column=[结算播放份额],
|
||||
”海田飞系“，所以我们需要column=[严选版权归属系], cell values = ['海田飞系'],所以有[严选版权归属系:('海田飞系')],
|
||||
”近3天“，所以我们需要column=[数据日期], cell values = [3],所以有[数据日期:(3)]""",
|
||||
"schemaLinks":"""["结算播放份额", "严选版权归属系":("'海田飞系'"), "数据日期":(3)]""",
|
||||
"sql":"""select 严选版权归属系, 结算播放份额 from 严选 where 严选版权归属系 = '海田飞系' and datediff('day', 数据日期, '2023-08-21') <= 3 """
|
||||
},
|
||||
{ "currentDate":"2023-05-22",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["是否潮流人歌曲", "C音歌曲ID", "C音歌曲MID", "歌曲名", "歌曲版本", "语种", "歌曲类型", "翻唱类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "结算播放量", "运营播放量", "付费用户结算播放量", "历史累计结算播放量", "运营搜播量", "结算搜播量", "运营完播量", "运营推播量", "近7日复播率", "日均搜播量", "数据日期"]""",
|
||||
"question":"对比近7天翻唱版和纯音乐的歌曲播放量",
|
||||
"priorSchemaLinks":"""['纯音乐'->语种, '翻唱版'->歌曲版本]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“对比近3天翻唱版和纯音乐的歌曲播放量“中，我们被问：
|
||||
“歌曲播放量”，所以我们需要column=[结算播放量]
|
||||
”翻唱版“，所以我们需要column=[歌曲版本], cell values = ['翻唱版'],所以有[歌曲版本:('翻唱版')]
|
||||
”和纯音乐的歌曲“，所以我们需要column=[语种], cell values = ['纯音乐'],所以有[语种:('纯音乐')]
|
||||
”近7天“，所以我们需要column=[数据日期], cell values = [7],所以有[数据日期:(7)]""",
|
||||
"schemaLinks":"""["结算播放量", "歌曲版本":("'翻唱版'"), "语种":("'纯音乐'"), "数据日期":(7)]""",
|
||||
"sql":"""select 歌曲版本, 语种, 结算播放量 from 歌曲库 where 歌曲版本 = '翻唱版' and 语种 = '纯音乐' and datediff('day', 数据日期, '2023-05-22') <= 7 """
|
||||
},
|
||||
{ "currentDate":"2023-05-31",
|
||||
"tableName":"艺人库",
|
||||
"fieldsList":"""["上下架状态", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "活跃区域", "年龄", "歌手才能", "歌手风格", "粉丝数", "潮音粉丝数", "超声波粉丝数", "推博粉丝数", "超声波歌曲数", "在架歌曲数", "超声波分享数", "独占歌曲数", "超声波在架歌曲评论数", "有播放量歌曲数", "数据日期"]""",
|
||||
"question":"对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数",
|
||||
"priorSchemaLinks":"""['1527896'->MPPM歌手ID, '1565463'->MPPM歌手ID, '2141459'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数“中，我们被问：
|
||||
“粉丝数”，所以我们需要column=[粉丝数]
|
||||
”陈拙悬、孟梅琦、赖媚韵“，所以我们需要column=[歌手名], cell values = ['陈拙悬', '孟梅琦', '赖媚韵'],所以有[歌手名:('陈拙悬', '孟梅琦', '赖媚韵')]""",
|
||||
"schemaLinks":"""["粉丝数", "歌手名":("'陈拙悬'", "'孟梅琦'", "'赖媚韵'")]""",
|
||||
"sql":"""select 歌手名, 粉丝数 from 艺人库 where 歌手名 in ('陈拙悬', '孟梅琦', '赖媚韵')"""
|
||||
},
|
||||
{ "currentDate":"2023-07-31",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"播放量大于1万的歌曲有多少",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“播放量大于1万的歌曲有多少“中，我们被问：
|
||||
“歌曲有多少”，所以我们需要column=[歌曲名]
|
||||
”播放量大于1万的“，所以我们需要column=[结算播放量], cell values = [10000],所以有[结算播放量:(10000)]""",
|
||||
"schemaLinks":"""["歌曲名", "结算播放量":(10000)]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 10000"""
|
||||
},
|
||||
{ "currentDate":"2023-07-31",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"内容库访问时长小于1小时，且来自美术部的用户是哪些",
|
||||
"priorSchemaLinks":"""['美术部'->部门]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库访问时长小于1小时，且来自美术部的用户是哪些“中，我们被问：
|
||||
“用户是哪些”，所以我们需要column=[用户名]
|
||||
”美术部的“，所以我们需要column=[部门], cell values = ['美术部'],所以有[部门:('美术部')]
|
||||
”访问时长小于1小时“，所以我们需要column=[访问时长], cell values = [1],所以有[访问时长:(1)]""",
|
||||
"schemaLinks":"""["用户名", "部门":("'美术部'"), "访问时长":(1)]""",
|
||||
"sql":"""select 用户名 from 内容库产品 where 部门 = '美术部' and 访问时长 < 1"""
|
||||
},
|
||||
{ "currentDate":"2023-08-31",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"内容库pv最高的用户有哪些",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库pv最高的用户有哪些“中，我们被问：
|
||||
“用户有哪些”，所以我们需要column=[用户名]
|
||||
”pv最高的“，所以我们需要column=[访问次数], cell values = [1],所以有[访问次数:(1)]""",
|
||||
"schemaLinks":"""["用户名", "访问次数":(1)]""",
|
||||
"sql":"""select 用户名 from 内容库产品 order by 访问次数 desc limit 1"""
|
||||
},
|
||||
{ "currentDate":"2023-08-31",
|
||||
"tableName":"艺人库",
|
||||
"fieldsList":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
|
||||
"question":"近90天袁亚伟播放量平均值是多少",
|
||||
"priorSchemaLinks":"""['152789226'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“近90天袁亚伟播放量平均值是多少“中，我们被问：
|
||||
“播放量平均值是多少”，所以我们需要column=[结算播放量]
|
||||
”袁亚伟“，所以我们需要column=[歌手名], cell values = ['袁亚伟'],所以有[歌手名:('袁亚伟')]
|
||||
”近90天“，所以我们需要column=[数据日期], cell values = [90],所以有[数据日期:(90)]""",
|
||||
"schemaLinks":"""["结算播放量", "歌手名":("'袁亚伟'"), "数据日期":(90)]""",
|
||||
"sql":"""select avg(结算播放量) from 艺人库 where 歌手名 = '袁亚伟' and datediff('day', 数据日期, '2023-08-31') <= 90 """
|
||||
},
|
||||
{ "currentDate":"2023-08-31",
|
||||
"tableName":"艺人库",
|
||||
"fieldsList":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
|
||||
"question":"周倩倩近7天结算播放量总和是多少",
|
||||
"priorSchemaLinks":"""['199509'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“周倩倩近7天结算播放量总和是多少“中，我们被问：
|
||||
“结算播放量总和是多少”，所以我们需要column=[结算播放量]
|
||||
”周倩倩“，所以我们需要column=[歌手名], cell values = ['周倩倩'],所以有[歌手名:('周倩倩')]
|
||||
”近7天“，所以我们需要column=[数据日期], cell values = [7],所以有[数据日期:(7)]""",
|
||||
"schemaLinks":"""["结算播放量", "歌手名":("'周倩倩'"), "数据日期":(7)]""",
|
||||
"sql":"""select sum(结算播放量) from 艺人库 where 歌手名 = '周倩倩' and datediff('day', 数据日期, '2023-08-31') <= 7 """
|
||||
},
|
||||
{ "currentDate":"2023-09-14",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
|
||||
"question":"内容库访问次数大于1k的部门是哪些",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库访问次数大于1k的部门是哪些“中，我们被问：
|
||||
“部门是哪些”，所以我们需要column=[部门]
|
||||
”访问次数大于1k的“，所以我们需要column=[访问次数], cell values = [1000],所以有[访问次数:(1000)]""",
|
||||
"schemaLinks":"""["部门", "访问次数":(1000)]""",
|
||||
"sql":"""select 部门 from 内容库产品 where 访问次数 > 1000"""
|
||||
},
|
||||
{ "currentDate":"2023-09-18",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "MPPM歌手ID", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"陈亿训唱的所有的播放量大于20k的孤勇者有哪些",
|
||||
"priorSchemaLinks":"""['199509'->MPPM歌手ID, '1527123'->MPPM歌曲ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“陈亿训唱的所有的播放量大于20k的孤勇者有哪些“中，我们被问：
|
||||
“孤勇者有哪些”，所以我们需要column=[歌曲名], cell values = ['孤勇者'],所以有[歌曲名:('孤勇者')]
|
||||
”播放量大于20k的“，所以我们需要column=[结算播放量], cell values = [20000],所以有[结算播放量:(20000)]
|
||||
”陈亿训唱的“，所以我们需要column=[歌手名], cell values = ['陈亿训'],所以有[歌手名:('陈亿训')]""",
|
||||
"schemaLinks":"""["歌曲名":("'孤勇者'"), "结算播放量":(20000), "歌手名":("'陈亿训'")]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 20000 and 歌手名 = '陈亿训' and 歌曲名 = '孤勇者'"""
|
||||
},
|
||||
{ "currentDate":"2023-09-18",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"周洁轮去年发布的歌曲有哪些",
|
||||
"priorSchemaLinks":"""['23109'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“周洁轮去年发布的歌曲有哪些“中，我们被问：
|
||||
“歌曲有哪些”，所以我们需要column=[歌曲名]
|
||||
”去年发布的“，所以我们需要column=[发布时间], cell values = [1],所以有[发布时间:(1)]
|
||||
”周洁轮“，所以我们需要column=[歌手名], cell values = ['周洁轮'],所以有[歌手名:('周洁轮')]""",
|
||||
"schemaLinks":"""["歌曲名", "发布时间":(1), "歌手名":("'周洁轮'")]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发布时间, '2023-09-18') <= 1 and 歌手名 = '周洁轮'"""
|
||||
},
|
||||
{ "currentDate":"2023-09-11",
|
||||
"tableName":"艺人库",
|
||||
"fieldsList":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "签约日期", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
|
||||
"question":"我想要近半年签约的播放量前十的歌手有哪些",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“我想要近半年签约的播放量前十的歌手“中，我们被问：
|
||||
“歌手有哪些”，所以我们需要column=[歌手名]
|
||||
”播放量前十的“，所以我们需要column=[结算播放量], cell values = [10],所以有[结算播放量:(10)]
|
||||
”近半年签约的“，所以我们需要column=[签约日期], cell values = [0.5],所以有[签约日期:(0.5)]""",
|
||||
"schemaLinks":"""["歌手名", "结算播放量":(10), "签约日期":(0.5)]""",
|
||||
"sql":"""select 歌手名 from 艺人库 where datediff('year', 签约日期, '2023-09-11') <= 0.5 order by 结算播放量 desc limit 10"""
|
||||
},
|
||||
{ "currentDate":"2023-08-12",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""",
|
||||
"question":"最近一年发行的歌曲中，有哪些在近7天播放超过一千万的",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“最近一年发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问：
|
||||
“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名]
|
||||
”最近一年发行的“，所以我们需要column=[发行日期], cell values = [1],所以有[发行日期:(1)]
|
||||
”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量], cell values = [7, 10000000],所以有[数据日期:(7), 结算播放量:(10000000)]""",
|
||||
"schemaLinks":"""["歌曲名", "发行日期":(1), "数据日期":(7), "结算播放量":(10000000)]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 1 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000"""
|
||||
},
|
||||
{ "currentDate":"2023-08-12",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""",
|
||||
"question":"今年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“今年以来发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问：
|
||||
“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名]
|
||||
”今年以来发行的“，所以我们需要column=[发行日期], cell values = [0],所以有[发行日期:(0)]
|
||||
”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量], cell values = [7, 10000000],所以有[数据日期:(7), 结算播放量:(10000000)]""",
|
||||
"schemaLinks":"""["歌曲名", "发行日期":(0), "数据日期":(7), "结算播放量":(10000000)]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 0 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000"""
|
||||
},
|
||||
{ "currentDate":"2023-08-12",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""",
|
||||
"question":"2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
|
||||
"priorSchemaLinks":"""['514129144'->MPPM歌曲ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问：
|
||||
“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名]
|
||||
”2023年以来发行的“，所以我们需要column=[发行日期], cell values = ['2023-01-01'],所以有[发行日期:('2023-01-01')]
|
||||
”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量], cell values = [7, 10000000],所以有[数据日期:(7), 结算播放量:(10000000)]""",
|
||||
"schemaLinks":"""["歌曲名", "发行日期":("'2023-01-01'"), "数据日期":(7), "结算播放量":(10000000)]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 发行日期 >= '2023-01-01' and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000"""
|
||||
},
|
||||
{ "currentDate":"2023-08-01",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"周洁轮2023年6月之后发布的歌曲有哪些",
|
||||
"priorSchemaLinks":"""['23109'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“周洁轮2023年6月之后发布的歌曲有哪些“中，我们被问：
|
||||
“歌曲有哪些”，所以我们需要column=[歌曲名]
|
||||
”2023年6月之后发布的“，所以我们需要column=[发布时间], cell values = ['2023-06-01'],所以有[发布时间:('2023-06-01')]
|
||||
”周洁轮“，所以我们需要column=[歌手名], cell values = ['周洁轮'],所以有[歌手名:('周洁轮')]""",
|
||||
"schemaLinks":"""["歌曲名", "发布时间":("'2023-06-01'"), "歌手名":("'周洁轮'")]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 发布时间 >= '2023-06-01' and 歌手名 = '周洁轮'"""
|
||||
},
|
||||
{ "currentDate":"2023-08-01",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？",
|
||||
"priorSchemaLinks":"""['2312311'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？“中，我们被问：
|
||||
“歌曲中，有哪些”，所以我们需要column=[歌曲名]
|
||||
“播放量大于500W的”，所以我们需要column=[结算播放量], cell values = [5000000],所以有[结算播放量:(5000000)]
|
||||
”邓梓琦在2023年1月5日之后发布的“，所以我们需要column=[发布时间], cell values = ['2023-01-05'],所以有[发布时间:('2023-01-05')]
|
||||
”邓梓琦“，所以我们需要column=[歌手名], cell values = ['邓梓琦'],所以有[歌手名:('邓梓琦')]""",
|
||||
"schemaLinks":"""["歌曲名", "结算播放量":(5000000), "发布时间":("'2023-01-05'"), "歌手名":("'邓梓琦'")]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 发布时间 >= '2023-01-05' and 歌手名 = '邓梓琦' and 结算播放量 > 5000000"""
|
||||
},
|
||||
{ "currentDate":"2023-09-17",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"2023年6月以后，张亮英播放量大于200万的歌曲有哪些？",
|
||||
"priorSchemaLinks":"""['45453'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“2023年6月以后，张亮英播放量大于200万的歌曲有哪些？“中，我们被问：
|
||||
“播放量大于200万的”，所以我们需要column=[结算播放量], cell values = [2000000],所以有[结算播放量:(2000000)]
|
||||
”2023年6月以后，张亮英“，所以我们需要column=[数据日期, 歌手名], cell values = ['2023-06-01', '张亮英'],所以有[数据日期:('2023-06-01'), 歌手名:('张亮英')],
|
||||
”歌曲有哪些“，所以我们需要column=[歌曲名]""",
|
||||
"schemaLinks":"""["结算播放量":(2000000), "数据日期":("'2023-06-01'"), "歌手名":("'张亮英'"), "歌曲名"]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 数据日期 >= '2023-06-01' and 歌手名 = '张亮英' and 结算播放量 > 2000000"""
|
||||
},
|
||||
{ "currentDate":"2023-08-16",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些",
|
||||
"priorSchemaLinks":"""['23109'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些“中，我们被问：
|
||||
“播放量大于20万的”，所以我们需要column=[结算播放量], cell values = [200000],所以有[结算播放量:(200000)]
|
||||
”2021年6月以后发布的“，所以我们需要column=[发布时间], cell values = ['2021-06-01'],所以有[发布时间:('2021-06-01')]
|
||||
”李雨纯“，所以我们需要column=[歌手名], cell values = ['李雨纯'],所以有[歌手名:('李雨纯')]""",
|
||||
"schemaLinks":"""["结算播放量":(200000), "发布时间":("'2021-06-01'"), "歌手名":("'李雨纯'")]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 发布时间 >= '2021-06-01' and 歌手名 = '李雨纯' and 结算播放量 > 200000"""
|
||||
},
|
||||
{ "currentDate":"2023-08-16",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
|
||||
"question":"刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些",
|
||||
"priorSchemaLinks":"""['4234234'->MPPM歌手ID]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些“中，我们被问：
|
||||
“播放量大于20万的”，所以我们需要column=[结算播放量], cell values = [200000],所以有[结算播放量:(200000)]
|
||||
”1992年4月2日到2020年5月2日之间发布的“， 所以我们需要column=[发布时间], cell values = ['1992-04-02', '2020-05-02'],所以有[发布时间:('1992-04-02', '2020-05-02')]
|
||||
”刘锝桦“，所以我们需要column=[歌手名], cell values = ['刘锝桦'],所以有[歌手名:('刘锝桦')]""",
|
||||
"schemaLinks":"""["结算播放量":(200000), "发布时间":("'1992-04-02'", "'2020-05-02'"), "歌手名":("'刘锝桦'")]""",
|
||||
"sql":"""select 歌曲名 from 歌曲库 where 发布时间 >= '1992-04-02' and 发布时间 <= '2020-05-02' and 歌手名 = '刘锝桦' and 结算播放量 > 200000"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-04",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"内容库近30天访问次数的平均数",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库近30天访问次数的平均数“中，我们被问：
|
||||
“访问次数的平均数”，所以我们需要column=[访问次数]
|
||||
”内容库近30天“，所以我们需要column=[数据日期], cell values = [30],所以有[数据日期:(30)]""",
|
||||
"schemaLinks":"""["访问次数", "数据日期":(30)]""",
|
||||
"sql":"""select avg(访问次数) from 内容库产品 where datediff('day', 数据日期, '2023-09-04') <= 30 """
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-04",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"内容库近半年哪个月的访问次数汇总最高",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库近半年哪个月的访问次数汇总最高“中，我们被问：
|
||||
“访问次数汇总最高”，所以我们需要column=[访问次数], cell values = [1],所以有[访问次数:(1)]
|
||||
”内容库近半年“，所以我们需要column=[数据日期], cell values = [0.5],所以有[数据日期:(0.5)]""",
|
||||
"schemaLinks":"""["访问次数":(1), "数据日期":(0.5)]""",
|
||||
"sql":"""select MONTH(数据日期), sum(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期) order by sum(访问次数) desc limit 1"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-04",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"内容库近半年每个月的平均访问次数",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库近半年每个月的平均访问次数“中，我们被问：
|
||||
“每个月的平均访问次数”，所以我们需要column=[访问次数]
|
||||
”内容库近半年“，所以我们需要column=[数据日期], cell values = [0.5],所以有[数据日期:(0.5)]""",
|
||||
"schemaLinks":"""["访问次数", "数据日期":(0.5)]""",
|
||||
"sql":"""select MONTH(数据日期), avg(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期)"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-10",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"内容库 按部门统计访问次数 top10 的部门",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“内容库 按部门统计访问次数 top10 的部门“中，我们被问：
|
||||
“访问次数 top10 的部门”，所以我们需要column=[访问次数], cell values = [10],所以有[访问次数:(10)]
|
||||
”内容库 按部门统计“，所以我们需要column=[部门]""",
|
||||
"schemaLinks":"""["访问次数":(10), "部门"]""",
|
||||
"sql":"""select 部门, sum(访问次数) from 内容库产品 group by 部门 order by sum(访问次数) desc limit 10"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-10",
|
||||
"tableName":"内容库产品",
|
||||
"fieldsList":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
|
||||
"question":"超音速 近7个月，月度总访问量超过 2万的月份",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“超音速 近7个月，月度总访问量超过 2万的月份“中，我们被问：
|
||||
“月度总访问量超过 2万的月份”，所以我们需要column=[访问次数], cell values = [20000],所以有[访问次数:(20000)]
|
||||
”超音速 近7个月“，所以我们需要column=[数据日期], cell values = [7],所以有[数据日期:(7)]""",
|
||||
"schemaLinks":"""["访问次数":(20000), "数据日期":(7)]""",
|
||||
"sql":"""select MONTH(数据日期) from 内容库产品 where datediff('day', 数据日期, '2023-09-10') <= 7 group by MONTH(数据日期) having sum(访问次数) > 20000"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-10",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲语言", "歌曲来源", "运营播放量", "播放量", "歌曲名", "结算播放量", "专辑名", "发布日期", "歌曲版本", "歌曲类型", "数据日期"]""",
|
||||
"question":"2022年7月到2023年7月之间发布到歌曲，按播放量取top 100，再按月粒度来统计近1年的运营播放量",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“2022年7月到2023年7月之间发布到歌曲，按播放量取top 100，再按月粒度来统计近1年的运营播放量“中，我们被问：
|
||||
“按月粒度来统计近1年的运营播放量”，所以我们需要column=[运营播放量, 数据日期], cell values = [1],所以有[运营播放量, 数据日期:(1)]
|
||||
”按播放量取top 100“，所以我们需要column=[播放量], cell values = [100],所以有[播放量:(100)]
|
||||
“2022年7月到2023年7月之间发布到歌曲”，所以我们需要column=[发布日期], cell values = ['2022-07-01', '2023-07-01'],所以有[发布日期:('2022-07-01', '2023-07-01')]""",
|
||||
"schemaLinks":"""["运营播放量", "数据日期":(1), "播放量":(100), "发布日期":("'2022-07-01'", "'2023-07-01'")]""",
|
||||
"sql":"""select MONTH(数据日期), sum(运营播放量) from (select 数据日期, 运营播放量 from 歌曲库 where 发布日期 >= '2022-07-01' and 发布日期 <= '2023-07-01' order by 播放量 desc limit 100) t where datediff('year', 数据日期, '2023-09-10') <= 1 group by MONTH(数据日期)"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-09-10",
|
||||
"tableName":"歌曲库",
|
||||
"fieldsList":"""["歌曲语言", "歌曲来源", "运营播放量", "播放量", "歌曲名", "结算播放量", "专辑名", "发布日期", "歌曲版本", "歌曲类型", "数据日期"]""",
|
||||
"question":"2022年7月到2023年7月之间发布到歌曲，按播放量取top100，再按月粒度来统计近1年的运营播放量之和，筛选出其中运营播放量之和大于2k的月份",
|
||||
"priorSchemaLinks":"""[]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“2022年7月到2023年7月之间发布到歌曲，按播放量取top100，再按月粒度来统计近1年的运营播放量之和，筛选出其中运营播放量之和大于2k的月份“中，我们被问：
|
||||
“筛选出其中运营播放量之和大于2k的月份”，所以我们需要column=[运营播放量], cell values = [2000],所以有[运营播放量:(2000)]
|
||||
”按月粒度来统计近1年的运营播放量之和“，所以我们需要column=[数据日期], cell values = [1],所以有[数据日期:(1)]
|
||||
”按播放量取top100“，所以我们需要column=[播放量], cell values = [100],所以有[播放量:(100)]
|
||||
”2022年7月到2023年7月之间发布到歌曲“，所以我们需要column=[发布日期], cell values = ['2022-07-01', '2023-07-01'],所以有[发布日期:('2022-07-01', '2023-07-01')]""",
|
||||
"schemaLinks":"""["运营播放量":(2000), "数据日期":(1), "播放量":(100), "发布日期":("'2022-07-01'", "'2023-07-01'")]""",
|
||||
"sql":"""select MONTH(数据日期), sum(运营播放量) from (select 数据日期, 运营播放量 from 歌曲库 where 发布日期 >= '2022-07-01' and 发布日期 <= '2023-07-01' order by 播放量 desc limit 100) t where datediff('year', 数据日期, '2023-09-10') <= 1 group by MONTH(数据日期) having sum(运营播放量) > 2000"""
|
||||
},
|
||||
{
|
||||
"currentDate":"2023-11-01",
|
||||
"tableName":"营销月模型",
|
||||
"fieldsList":"""["国家中文名", "机型类别", "销量", "数据日期"]""",
|
||||
"question":"今年智能机在哪个国家的销量之和最高",
|
||||
"priorSchemaLinks":"""['智能机'->机型类别]""",
|
||||
"analysis": """让我们一步一步地思考。在问题“今年智能机在哪个国家的销量之和最高“中，我们被问：
|
||||
“销量最高”，所以我们需要column=[销量], cell values = [1],所以有[销量:(1)]
|
||||
”今年“，所以我们需要column=[数据日期], cell values = ['2023-01-01', '2023-11-01'],所以有[数据日期:('2023-01-01', '2023-11-01')]
|
||||
”智能机“，所以我们需要column=[机型类别], cell values = ['智能机'],所以有[机型类别:('智能机')]""",
|
||||
"schemaLinks":"""["销量":(1), "数据日期":("'2023-01-01'", "'2023-11-01'"), "机型类别":("'智能机'")]""",
|
||||
"sql":"""select 国家中文名, sum(销量) from 营销月模型 where 机型类别 = '智能机' and 数据日期 >= '2023-01-01' and 数据日期 <= '2023-11-01' group by 国家中文名 order by sum(销量) desc limit 1"""
|
||||
}
|
||||
]
|
||||
@@ -36,12 +36,12 @@ CHROMA_DB_PERSIST_DIR = config.get(chroma_db_section_name, 'CHROMA_DB_PERSIST_DI
|
||||
PRESET_QUERY_COLLECTION_NAME = config.get(chroma_db_section_name, 'PRESET_QUERY_COLLECTION_NAME')
|
||||
SOLVED_QUERY_COLLECTION_NAME = config.get(chroma_db_section_name, 'SOLVED_QUERY_COLLECTION_NAME')
|
||||
TEXT2DSLAGENT_COLLECTION_NAME = config.get(chroma_db_section_name, 'TEXT2DSLAGENT_COLLECTION_NAME')
|
||||
TEXT2DSLAGENTCS_COLLECTION_NAME = config.get(chroma_db_section_name, 'TEXT2DSLAGENTCS_COLLECTION_NAME')
|
||||
TEXT2DSLAGENTACT_COLLECTION_NAME = config.get(chroma_db_section_name, 'TEXT2DSLAGENTACT_COLLECTION_NAME')
|
||||
TEXT2DSL_EXAMPLE_NUM = int(config.get(chroma_db_section_name, 'TEXT2DSL_EXAMPLE_NUM'))
|
||||
TEXT2DSL_FEWSHOTS_NUM = int(config.get(chroma_db_section_name, 'TEXT2DSL_FEWSHOTS_NUM'))
|
||||
TEXT2DSL_SELF_CONSISTENCY_NUM = int(config.get(chroma_db_section_name, 'TEXT2DSL_SELF_CONSISTENCY_NUM'))
|
||||
TEXT2DSL_IS_SHORTCUT = eval(config.get(chroma_db_section_name, 'TEXT2DSL_IS_SHORTCUT'))
|
||||
TEXT2DSL_IS_SELF_CONSISTENCY = eval(config.get(chroma_db_section_name, 'TEXT2DSL_IS_SELF_CONSISTENCY'))
|
||||
ACT_MIN_WINDOWN_SIZE = int(config.get(chroma_db_section_name, 'ACT_MIN_WINDOWN_SIZE'))
|
||||
ACT_MAX_WINDOWN_SIZE = int(config.get(chroma_db_section_name, 'ACT_MAX_WINDOWN_SIZE'))
|
||||
CHROMA_DB_PERSIST_PATH = os.path.join(PROJECT_DIR_PATH, CHROMA_DB_PERSIST_DIR)
|
||||
|
||||
text2vec_section_name = "Text2Vec"
|
||||
@@ -63,8 +63,15 @@ if __name__ == "__main__":
|
||||
logger.info(f"LLMPARSER_HOST: {LLMPARSER_HOST}")
|
||||
logger.info(f"LLMPARSER_PORT: {LLMPARSER_PORT}")
|
||||
logger.info(f"llm_config_dict: {llm_config_dict}")
|
||||
logger.info(f"LLM_PROVIDER_NAME: {LLM_PROVIDER_NAME}")
|
||||
logger.info(f"PRESET_QUERY_COLLECTION_NAME: {PRESET_QUERY_COLLECTION_NAME}")
|
||||
logger.info(f"SOLVED_QUERY_COLLECTION_NAME: {SOLVED_QUERY_COLLECTION_NAME}")
|
||||
logger.info(f"TEXT2DSLAGENT_COLLECTION_NAME: {TEXT2DSLAGENT_COLLECTION_NAME}")
|
||||
logger.info(f"TEXT2DSLAGENTACT_COLLECTION_NAME: {TEXT2DSLAGENTACT_COLLECTION_NAME}")
|
||||
logger.info(f"TEXT2DSL_EXAMPLE_NUM: {TEXT2DSL_EXAMPLE_NUM}")
|
||||
logger.info(f"TEXT2DSL_FEWSHOTS_NUM: {TEXT2DSL_FEWSHOTS_NUM}")
|
||||
logger.info(f"TEXT2DSL_SELF_CONSISTENCY_NUM: {TEXT2DSL_SELF_CONSISTENCY_NUM}")
|
||||
logger.info(f"TEXT2DSL_IS_SHORTCUT: {TEXT2DSL_IS_SHORTCUT}")
|
||||
logger.info(f"TEXT2DSL_IS_SELF_CONSISTENCY: {TEXT2DSL_IS_SELF_CONSISTENCY}")
|
||||
logger.info(f"ACT_MIN_WINDOWN_SIZE: {ACT_MIN_WINDOWN_SIZE}")
|
||||
logger.info(f"ACT_MAX_WINDOWN_SIZE: {ACT_MAX_WINDOWN_SIZE}")
|
||||
|
||||
|
||||
@@ -7,12 +7,12 @@ CHROMA_DB_PERSIST_DIR = chm_db
|
||||
PRESET_QUERY_COLLECTION_NAME = preset_query_collection
|
||||
SOLVED_QUERY_COLLECTION_NAME = solved_query_collection
|
||||
TEXT2DSLAGENT_COLLECTION_NAME = text2dsl_agent_collection
|
||||
TEXT2DSLAGENTCS_COLLECTION_NAME = text2dsl_agent_cs_collection
|
||||
TEXT2DSLAGENTACT_COLLECTION_NAME = text2dsl_agent_act_collection
|
||||
TEXT2DSL_EXAMPLE_NUM = 15
|
||||
TEXT2DSL_FEWSHOTS_NUM = 10
|
||||
TEXT2DSL_SELF_CONSISTENCY_NUM = 5
|
||||
TEXT2DSL_IS_SHORTCUT = False
|
||||
TEXT2DSL_IS_SELF_CONSISTENCY = False
|
||||
ACT_MIN_WINDOWN_SIZE = 6
|
||||
ACT_MAX_WINDOWN_SIZE = 10
|
||||
|
||||
[Text2Vec]
|
||||
HF_TEXT2VEC_MODEL_NAME = GanymedeNil/text2vec-large-chinese
|
||||
|
||||
250
chat/python/few_shot_example/s2ql_examplar3_transformed.json
Normal file
250
chat/python/few_shot_example/s2ql_examplar3_transformed.json
Normal file
@@ -0,0 +1,250 @@
|
||||
[
|
||||
{
|
||||
"question": "比较jackjchen和robinlee在内容库的访问次数",
|
||||
"questionAugmented": "比较jackjchen和robinlee在内容库的访问次数 (补充信息:’'jackjchen'‘是一个’用户名‘，’ 'robinlee'‘是一个’用户名‘。当前的日期是2020-12-01) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"部门\", \"模块\", \"用户名\", \"访问次数\", \"访问人数\", \"访问时长\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 用户名, 访问次数 from 内容库产品 where 用户名 in ('jackjchen', 'robinlee')",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"比较jackjchen和robinlee在内容库的访问次数 (补充信息:’'jackjchen'‘是一个’用户名‘，’ 'robinlee'‘是一个’用户名‘。当前的日期是2020-12-01) (备注: )\", we are asked:\n\"’用户名‘，\" so we need column = [用户名]\n\"的访问次数 \" so we need column = [访问次数]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [jackjchen,robinlee]. So the Schema_links are:\nSchema_links: [用户名,访问次数,jackjchen,robinlee]",
|
||||
"generatedSchemaLinkings": "[用户名,访问次数,jackjchen,robinlee]"
|
||||
},
|
||||
{
|
||||
"question": "内容库近12个月访问人数 按部门",
|
||||
"questionAugmented": "内容库近12个月访问人数 按部门 (补充信息:。当前的日期是2022-11-06) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"部门\", \"模块\", \"用户名\", \"访问次数\", \"访问人数\", \"访问时长\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 部门, 数据日期, 访问人数 from 内容库产品 where datediff('month', 数据日期, '2022-11-06') <= 12 ",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库近12个月访问人数 按部门 (补充信息:。当前的日期是2022-11-06) (备注: )\", we are asked:\n\" 按部门 (\" so we need column = [部门]\n\"访问人数 按\" so we need column = [访问人数]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [12,month]. So the Schema_links are:\nSchema_links: [部门,访问人数,数据日期,12,month]",
|
||||
"generatedSchemaLinkings": "[部门,访问人数,数据日期,12,month]"
|
||||
},
|
||||
{
|
||||
"question": "内容库美术部、技术研发部的访问时长",
|
||||
"questionAugmented": "内容库美术部、技术研发部的访问时长 (补充信息:’'美术部'‘是一个’部门‘，’ '技术研发部'‘是一个’部门‘。当前的日期是2023-04-21) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"部门\", \"模块\", \"用户名\", \"访问次数\", \"访问人数\", \"访问时长\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 部门, 访问时长 from 内容库产品 where 部门 in ('美术部', '技术研发部')",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库美术部、技术研发部的访问时长 (补充信息:’'美术部'‘是一个’部门‘，’ '技术研发部'‘是一个’部门‘。当前的日期是2023-04-21) (备注: )\", we are asked:\n\"部门‘，’ \" so we need column = [部门]\n\"的访问时长 \" so we need column = [访问时长]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [美术部,技术研发部]. So the Schema_links are:\nSchema_links: [部门,访问时长,美术部,技术研发部]",
|
||||
"generatedSchemaLinkings": "[部门,访问时长,美术部,技术研发部]"
|
||||
},
|
||||
{
|
||||
"question": "近3天海田飞系MPPM结算播放份额",
|
||||
"questionAugmented": "近3天海田飞系MPPM结算播放份额 (补充信息:’'海田飞系'‘是一个’严选版权归属系‘。当前的日期是2023-08-21) (备注: )",
|
||||
"dbSchema": "Table: 严选, Columns = [\"严选版权归属系\", \"付费模式\", \"结算播放份额\", \"付费用户结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 严选版权归属系, 结算播放份额 from 严选 where 严选版权归属系 = '海田飞系' and datediff('day', 数据日期, '2023-08-21') <= 3 ",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"近3天海田飞系MPPM结算播放份额 (补充信息:’'海田飞系'‘是一个’严选版权归属系‘。当前的日期是2023-08-21) (备注: )\", we are asked:\n\"结算播放份额 \" so we need column = [结算播放份额]\n\"严选版权归属系\" so we need column = [严选版权归属系]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [海田飞系,3,day]. So the Schema_links are:\nSchema_links: [结算播放份额,严选版权归属系,数据日期,海田飞系,3,day]",
|
||||
"generatedSchemaLinkings": "[结算播放份额,严选版权归属系,数据日期,海田飞系,3,day]"
|
||||
},
|
||||
{
|
||||
"question": "对比近7天翻唱版和纯音乐的歌曲播放量",
|
||||
"questionAugmented": "对比近7天翻唱版和纯音乐的歌曲播放量 (补充信息:’'纯音乐'‘是一个’语种‘，’ '翻唱版'‘是一个’歌曲版本‘。当前的日期是2023-05-22) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"是否潮流人歌曲\", \"C音歌曲ID\", \"C音歌曲MID\", \"歌曲名\", \"歌曲版本\", \"语种\", \"歌曲类型\", \"翻唱类型\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"结算播放量\", \"运营播放量\", \"付费用户结算播放量\", \"历史累计结算播放量\", \"运营搜播量\", \"结算搜播量\", \"运营完播量\", \"运营推播量\", \"近7日复播率\", \"日均搜播量\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲版本, 语种, 结算播放量 from 歌曲库 where 歌曲版本 = '翻唱版' and 语种 = '纯音乐' and datediff('day', 数据日期, '2023-05-22') <= 7 ",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"对比近7天翻唱版和纯音乐的歌曲播放量 (补充信息:’'纯音乐'‘是一个’语种‘，’ '翻唱版'‘是一个’歌曲版本‘。当前的日期是2023-05-22) (备注: )\", we are asked:\n\"曲播放量 (\" so we need column = [结算播放量]\n\"’歌曲版本‘\" so we need column = [歌曲版本]\n\"语种‘，’ \" so we need column = [语种]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [7,翻唱版,纯音乐,day]. So the Schema_links are:\nSchema_links: [结算播放量,歌曲版本,语种,数据日期,7,翻唱版,纯音乐,day]",
|
||||
"generatedSchemaLinkings": "[结算播放量,歌曲版本,语种,数据日期,7,翻唱版,纯音乐,day]"
|
||||
},
|
||||
{
|
||||
"question": "对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数",
|
||||
"questionAugmented": "对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数 (补充信息:’'1527896'‘是一个’MPPM歌手ID‘，’ '1565463'‘是一个’MPPM歌手ID‘，’ '2141459'‘是一个’MPPM歌手ID‘。当前的日期是2023-05-31) (备注: )",
|
||||
"dbSchema": "Table: 艺人库, Columns = [\"上下架状态\", \"歌手名\", \"歌手等级\", \"歌手类型\", \"歌手来源\", \"MPPM潮流人等级\", \"活跃区域\", \"年龄\", \"歌手才能\", \"歌手风格\", \"粉丝数\", \"潮音粉丝数\", \"超声波粉丝数\", \"推博粉丝数\", \"超声波歌曲数\", \"在架歌曲数\", \"超声波分享数\", \"独占歌曲数\", \"超声波在架歌曲评论数\", \"有播放量歌曲数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌手名, 粉丝数 from 艺人库 where 歌手名 in ('陈拙悬', '孟梅琦', '赖媚韵')",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数 (补充信息:’'1527896'‘是一个’MPPM歌手ID‘，’ '1565463'‘是一个’MPPM歌手ID‘，’ '2141459'‘是一个’MPPM歌手ID‘。当前的日期是2023-05-31) (备注: )\", we are asked:\n\"歌手ID‘，\" so we need column = [歌手名]\n\"的粉丝数 (\" so we need column = [粉丝数]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [陈拙悬,孟梅琦,赖媚韵]. So the Schema_links are:\nSchema_links: [歌手名,粉丝数,陈拙悬,孟梅琦,赖媚韵]",
|
||||
"generatedSchemaLinkings": "[歌手名,粉丝数,陈拙悬,孟梅琦,赖媚韵]"
|
||||
},
|
||||
{
|
||||
"question": "播放量大于1万的歌曲有多少",
|
||||
"questionAugmented": "播放量大于1万的歌曲有多少 (补充信息:。当前的日期是2023-07-31) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌曲类型\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 结算播放量 > 10000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"播放量大于1万的歌曲有多少 (补充信息:。当前的日期是2023-07-31) (备注: )\", we are asked:\n\"歌曲有多少 \" so we need column = [歌曲名]\n\"播放量大于1\" so we need column = [结算播放量]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [10000]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,10000]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,10000]"
|
||||
},
|
||||
{
|
||||
"question": "内容库访问时长小于1小时，且来自美术部的用户是哪些",
|
||||
"questionAugmented": "内容库访问时长小于1小时，且来自美术部的用户是哪些 (补充信息:’'美术部'‘是一个’部门‘。当前的日期是2023-07-31) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 用户名 from 内容库产品 where 部门 = '美术部' and 访问时长 < 1",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库访问时长小于1小时，且来自美术部的用户是哪些 (补充信息:’'美术部'‘是一个’部门‘。当前的日期是2023-07-31) (备注: )\", we are asked:\n\"术部的用户是\" so we need column = [用户名]\n\"一个’部门‘\" so we need column = [部门]\n\"访问时长小于\" so we need column = [访问时长]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [美术部,1]. So the Schema_links are:\nSchema_links: [用户名,部门,访问时长,美术部,1]",
|
||||
"generatedSchemaLinkings": "[用户名,部门,访问时长,美术部,1]"
|
||||
},
|
||||
{
|
||||
"question": "内容库pv最高的用户有哪些",
|
||||
"questionAugmented": "内容库pv最高的用户有哪些 (补充信息:。当前的日期是2023-08-31) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 用户名 from 内容库产品 order by 访问次数 desc limit 1",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库pv最高的用户有哪些 (补充信息:。当前的日期是2023-08-31) (备注: )\", we are asked:\n\"用户有哪些 (\" so we need column = [用户名]\n\"最高的用户有\" so we need column = [访问次数]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1]. So the Schema_links are:\nSchema_links: [用户名,访问次数,1]",
|
||||
"generatedSchemaLinkings": "[用户名,访问次数,1]"
|
||||
},
|
||||
{
|
||||
"question": "近90天袁亚伟播放量平均值是多少",
|
||||
"questionAugmented": "近90天袁亚伟播放量平均值是多少 (补充信息:’'152789226'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-31) (备注: )",
|
||||
"dbSchema": "Table: 艺人库, Columns = [\"播放量层级\", \"播放量单调性\", \"播放量方差\", \"播放量突增类型\", \"播放量集中度\", \"歌手名\", \"歌手等级\", \"歌手类型\", \"歌手来源\", \"MPPM潮流人等级\", \"结算播放量\", \"运营播放量\", \"历史累计结算播放量\", \"有播放量歌曲数\", \"历史累计运营播放量\", \"付费用户结算播放量\", \"结算播放量占比\", \"运营播放份额\", \"免费用户结算播放占比\", \"完播量\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select avg(结算播放量) from 艺人库 where 歌手名 = '袁亚伟' and datediff('day', 数据日期, '2023-08-31') <= 90 ",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"近90天袁亚伟播放量平均值是多少 (补充信息:’'152789226'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-31) (备注: )\", we are asked:\n\"播放量平均值\" so we need column = [结算播放量]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [袁亚伟,90,day]. So the Schema_links are:\nSchema_links: [结算播放量,歌手名,数据日期,袁亚伟,90,day]",
|
||||
"generatedSchemaLinkings": "[结算播放量,歌手名,数据日期,袁亚伟,90,day]"
|
||||
},
|
||||
{
|
||||
"question": "周倩倩近7天结算播放量总和是多少",
|
||||
"questionAugmented": "周倩倩近7天结算播放量总和是多少 (补充信息:’'199509'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-31) (备注: )",
|
||||
"dbSchema": "Table: 艺人库, Columns = [\"播放量层级\", \"播放量单调性\", \"播放量方差\", \"播放量突增类型\", \"播放量集中度\", \"歌手名\", \"歌手等级\", \"歌手类型\", \"歌手来源\", \"MPPM潮流人等级\", \"结算播放量\", \"运营播放量\", \"历史累计结算播放量\", \"有播放量歌曲数\", \"历史累计运营播放量\", \"付费用户结算播放量\", \"结算播放量占比\", \"运营播放份额\", \"免费用户结算播放占比\", \"完播量\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select sum(结算播放量) from 艺人库 where 歌手名 = '周倩倩' and datediff('day', 数据日期, '2023-08-31') <= 7 ",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"周倩倩近7天结算播放量总和是多少 (补充信息:’'199509'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-31) (备注: )\", we are asked:\n\"结算播放量总\" so we need column = [结算播放量]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [周倩倩,7,day]. So the Schema_links are:\nSchema_links: [结算播放量,歌手名,数据日期,周倩倩,7,day]",
|
||||
"generatedSchemaLinkings": "[结算播放量,歌手名,数据日期,周倩倩,7,day]"
|
||||
},
|
||||
{
|
||||
"question": "内容库访问次数大于1k的部门是哪些",
|
||||
"questionAugmented": "内容库访问次数大于1k的部门是哪些 (补充信息:。当前的日期是2023-09-14) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"部门\", \"模块\", \"用户名\", \"访问次数\", \"访问人数\", \"访问时长\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 部门 from 内容库产品 where 访问次数 > 1000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库访问次数大于1k的部门是哪些 (补充信息:。当前的日期是2023-09-14) (备注: )\", we are asked:\n\"访问次数大于\" so we need column = [访问次数]\n\"部门是哪些 \" so we need column = [部门]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1000]. So the Schema_links are:\nSchema_links: [访问次数,部门,1000]",
|
||||
"generatedSchemaLinkings": "[访问次数,部门,1000]"
|
||||
},
|
||||
{
|
||||
"question": "陈亿训唱的所有的播放量大于20k的孤勇者有哪些",
|
||||
"questionAugmented": "陈亿训唱的所有的播放量大于20k的孤勇者有哪些 (补充信息:’'199509'‘是一个’MPPM歌手ID‘，’ '1527123'‘是一个’MPPM歌曲ID‘。当前的日期是2023-09-18) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"MPPM歌手ID\", \"歌曲版本\", \"歌曲类型\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 结算播放量 > 20000 and 歌手名 = '陈亿训' and 歌曲名 = '孤勇者'",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"陈亿训唱的所有的播放量大于20k的孤勇者有哪些 (补充信息:’'199509'‘是一个’MPPM歌手ID‘，’ '1527123'‘是一个’MPPM歌曲ID‘。当前的日期是2023-09-18) (备注: )\", we are asked:\n\"歌曲ID‘。\" so we need column = [歌曲名]\n\"的所有的播放量\" so we need column = [结算播放量]\n\"歌手ID‘，\" so we need column = [歌手名]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [孤勇者,20000,陈亿训]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,歌手名,孤勇者,20000,陈亿训]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,歌手名,孤勇者,20000,陈亿训]"
|
||||
},
|
||||
{
|
||||
"question": "周洁轮去年发布的歌曲有哪些",
|
||||
"questionAugmented": "周洁轮去年发布的歌曲有哪些 (补充信息:’'23109'‘是一个’MPPM歌手ID‘。当前的日期是2023-09-18) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌手名\", \"歌曲类型\", \"发布时间\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where datediff('year', 发布时间, '2023-09-18') <= 1 and 歌手名 = '周洁轮'",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"周洁轮去年发布的歌曲有哪些 (补充信息:’'23109'‘是一个’MPPM歌手ID‘。当前的日期是2023-09-18) (备注: )\", we are asked:\n\"歌曲有哪些 \" so we need column = [歌曲名]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"发布的歌曲有\" so we need column = [发布时间]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1,周洁轮,year]. So the Schema_links are:\nSchema_links: [歌曲名,歌手名,发布时间,1,周洁轮,year]",
|
||||
"generatedSchemaLinkings": "[歌曲名,歌手名,发布时间,1,周洁轮,year]"
|
||||
},
|
||||
{
|
||||
"question": "我想要近半年签约的播放量前十的歌手有哪些",
|
||||
"questionAugmented": "我想要近半年签约的播放量前十的歌手有哪些 (补充信息:。当前的日期是2023-09-11) (备注: )",
|
||||
"dbSchema": "Table: 艺人库, Columns = [\"播放量层级\", \"播放量单调性\", \"播放量方差\", \"播放量突增类型\", \"播放量集中度\", \"歌手名\", \"歌手等级\", \"歌手类型\", \"歌手来源\", \"签约日期\", \"MPPM潮流人等级\", \"结算播放量\", \"运营播放量\", \"历史累计结算播放量\", \"有播放量歌曲数\", \"历史累计运营播放量\", \"付费用户结算播放量\", \"结算播放量占比\", \"运营播放份额\", \"免费用户结算播放占比\", \"完播量\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌手名 from 艺人库 where datediff('year', 签约日期, '2023-09-11') <= 0.5 order by 结算播放量 desc limit 10",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"我想要近半年签约的播放量前十的歌手有哪些 (补充信息:。当前的日期是2023-09-11) (备注: )\", we are asked:\n\"歌手有哪些 \" so we need column = [歌手名]\n\"签约的播放量\" so we need column = [结算播放量]\n\"签约的播放量\" so we need column = [签约日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [10,0.5,year]. So the Schema_links are:\nSchema_links: [歌手名,结算播放量,签约日期,10,0.5,year]",
|
||||
"generatedSchemaLinkings": "[歌手名,结算播放量,签约日期,10,0.5,year]"
|
||||
},
|
||||
{
|
||||
"question": "最近一年发行的歌曲中，有哪些在近7天播放超过一千万的",
|
||||
"questionAugmented": "最近一年发行的歌曲中，有哪些在近7天播放超过一千万的 (补充信息:。当前的日期是2023-08-12) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"发行日期\", \"歌曲语言\", \"歌曲来源\", \"歌曲流派\", \"歌曲名\", \"歌曲版本\", \"歌曲类型\", \"发行时间\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 1 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"最近一年发行的歌曲中，有哪些在近7天播放超过一千万的 (补充信息:。当前的日期是2023-08-12) (备注: )\", we are asked:\n\"的歌曲中，有\" so we need column = [歌曲名]\n\"天播放超过一\" so we need column = [结算播放量]\n\"最近一年发行\" so we need column = [发行日期]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [10000000,1,7,year,day]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,发行日期,数据日期,10000000,1,7,year,day]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,发行日期,数据日期,10000000,1,7,year,day]"
|
||||
},
|
||||
{
|
||||
"question": "今年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
|
||||
"questionAugmented": "今年以来发行的歌曲中，有哪些在近7天播放超过一千万的 (补充信息:。当前的日期是2023-08-12) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"发行日期\", \"歌曲语言\", \"歌曲来源\", \"歌曲流派\", \"歌曲名\", \"歌曲版本\", \"歌曲类型\", \"发行时间\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 0 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"今年以来发行的歌曲中，有哪些在近7天播放超过一千万的 (补充信息:。当前的日期是2023-08-12) (备注: )\", we are asked:\n\"的歌曲中，有\" so we need column = [歌曲名]\n\"天播放超过一\" so we need column = [结算播放量]\n\"年以来发行的\" so we need column = [发行日期]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [10000000,0,7,year,day]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,发行日期,数据日期,10000000,0,7,year,day]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,发行日期,数据日期,10000000,0,7,year,day]"
|
||||
},
|
||||
{
|
||||
"question": "2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
|
||||
"questionAugmented": "2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的 (补充信息:’'514129144'‘是一个’MPPM歌曲ID‘。当前的日期是2023-08-12) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"发行日期\", \"歌曲语言\", \"歌曲来源\", \"歌曲流派\", \"歌曲名\", \"歌曲版本\", \"歌曲类型\", \"发行时间\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 发行日期 >= '2023-01-01' and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的 (补充信息:’'514129144'‘是一个’MPPM歌曲ID‘。当前的日期是2023-08-12) (备注: )\", we are asked:\n\"的歌曲中，有\" so we need column = [歌曲名]\n\"天播放超过一\" so we need column = [结算播放量]\n\"年以来发行的\" so we need column = [发行日期]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [10000000,2023-01-01,7,day]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,发行日期,数据日期,10000000,2023-01-01,7,day]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,发行日期,数据日期,10000000,2023-01-01,7,day]"
|
||||
},
|
||||
{
|
||||
"question": "周洁轮2023年6月之后发布的歌曲有哪些",
|
||||
"questionAugmented": "周洁轮2023年6月之后发布的歌曲有哪些 (补充信息:’'23109'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-01) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌手名\", \"歌曲类型\", \"发布时间\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 发布时间 >= '2023-06-01' and 歌手名 = '周洁轮'",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"周洁轮2023年6月之后发布的歌曲有哪些 (补充信息:’'23109'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-01) (备注: )\", we are asked:\n\"歌曲有哪些 \" so we need column = [歌曲名]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"月之后发布的\" so we need column = [发布时间]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [2023-06-01,周洁轮]. So the Schema_links are:\nSchema_links: [歌曲名,歌手名,发布时间,2023-06-01,周洁轮]",
|
||||
"generatedSchemaLinkings": "[歌曲名,歌手名,发布时间,2023-06-01,周洁轮]"
|
||||
},
|
||||
{
|
||||
"question": "邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？",
|
||||
"questionAugmented": "邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？ (补充信息:’'2312311'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-01) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌手名\", \"歌曲类型\", \"发布时间\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 发布时间 >= '2023-01-05' and 歌手名 = '邓梓琦' and 结算播放量 > 5000000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？ (补充信息:’'2312311'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-01) (备注: )\", we are asked:\n\"的歌曲中，有\" so we need column = [歌曲名]\n\"中，有哪些播放量\" so we need column = [结算播放量]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"日之后发布的\" so we need column = [发布时间]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [5000000,2023-01-05,邓梓琦]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,歌手名,发布时间,5000000,2023-01-05,邓梓琦]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,歌手名,发布时间,5000000,2023-01-05,邓梓琦]"
|
||||
},
|
||||
{
|
||||
"question": "2023年6月以后，张亮英播放量大于200万的歌曲有哪些？",
|
||||
"questionAugmented": "2023年6月以后，张亮英播放量大于200万的歌曲有哪些？ (补充信息:’'45453'‘是一个’MPPM歌手ID‘。当前的日期是2023-09-17) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌手名\", \"歌曲类型\", \"发布时间\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 数据日期 >= '2023-06-01' and 歌手名 = '张亮英' and 结算播放量 > 2000000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"2023年6月以后，张亮英播放量大于200万的歌曲有哪些？ (补充信息:’'45453'‘是一个’MPPM歌手ID‘。当前的日期是2023-09-17) (备注: )\", we are asked:\n\"的歌曲有哪些？ (\" so we need column = [歌曲名]\n\"后，张亮英播放量大\" so we need column = [结算播放量]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [2000000,2023-06-01,张亮英]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,歌手名,数据日期,2000000,2023-06-01,张亮英]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,歌手名,数据日期,2000000,2023-06-01,张亮英]"
|
||||
},
|
||||
{
|
||||
"question": "2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些",
|
||||
"questionAugmented": "2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些 (补充信息:’'23109'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-16) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌手名\", \"歌曲类型\", \"发布时间\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 发布时间 >= '2021-06-01' and 歌手名 = '李雨纯' and 结算播放量 > 200000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些 (补充信息:’'23109'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-16) (备注: )\", we are asked:\n\"歌曲有哪些 \" so we need column = [歌曲名]\n\"的播放量大于\" so we need column = [结算播放量]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"月以后发布的\" so we need column = [发布时间]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [200000,2021-06-01,李雨纯]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,歌手名,发布时间,200000,2021-06-01,李雨纯]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,歌手名,发布时间,200000,2021-06-01,李雨纯]"
|
||||
},
|
||||
{
|
||||
"question": "刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些",
|
||||
"questionAugmented": "刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些 (补充信息:’'4234234'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-16) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲名\", \"歌曲版本\", \"歌手名\", \"歌曲类型\", \"发布时间\", \"MPPM歌曲ID\", \"是否严选窄口径歌曲\", \"是否严选宽口径歌曲\", \"是否潮流人歌曲\", \"超声波歌曲ID\", \"C音歌曲ID\", \"C音歌曲MID\", \"结算播放量\", \"运营播放量\", \"分享量\", \"收藏量\", \"运营搜播量\", \"结算搜播量\", \"拉新用户数\", \"拉活用户数\", \"分享率\", \"结算播放份额\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 歌曲名 from 歌曲库 where 发布时间 >= '1992-04-02' and 发布时间 <= '2020-05-02' and 歌手名 = '刘锝桦' and 结算播放量 > 200000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些 (补充信息:’'4234234'‘是一个’MPPM歌手ID‘。当前的日期是2023-08-16) (备注: )\", we are asked:\n\"歌曲有哪些 \" so we need column = [歌曲名]\n\"发布的播放量\" so we need column = [结算播放量]\n\"歌手ID‘。\" so we need column = [歌手名]\n\"日之间发布的\" so we need column = [发布时间]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [200000,刘锝桦,1992-04-02,2020-05-02]. So the Schema_links are:\nSchema_links: [歌曲名,结算播放量,歌手名,发布时间,200000,刘锝桦,1992-04-02,2020-05-02]",
|
||||
"generatedSchemaLinkings": "[歌曲名,结算播放量,歌手名,发布时间,200000,刘锝桦,1992-04-02,2020-05-02]"
|
||||
},
|
||||
{
|
||||
"question": "内容库近30天访问次数的平均数",
|
||||
"questionAugmented": "内容库近30天访问次数的平均数 (补充信息:。当前的日期是2023-09-04) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select avg(访问次数) from 内容库产品 where datediff('day', 数据日期, '2023-09-04') <= 30 ",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库近30天访问次数的平均数 (补充信息:。当前的日期是2023-09-04) (备注: )\", we are asked:\n\"访问次数的平均数\" so we need column = [访问次数]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [30,day]. So the Schema_links are:\nSchema_links: [访问次数,数据日期,30,day]",
|
||||
"generatedSchemaLinkings": "[访问次数,数据日期,30,day]"
|
||||
},
|
||||
{
|
||||
"question": "内容库近半年哪个月的访问次数汇总最高",
|
||||
"questionAugmented": "内容库近半年哪个月的访问次数汇总最高 (补充信息:。当前的日期是2023-09-04) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select MONTH(数据日期), sum(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期) order by sum(访问次数) desc limit 1",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库近半年哪个月的访问次数汇总最高 (补充信息:。当前的日期是2023-09-04) (备注: )\", we are asked:\n\"的访问次数汇总\" so we need column = [访问次数]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1,0.5,year]. So the Schema_links are:\nSchema_links: [访问次数,数据日期,1,0.5,year]",
|
||||
"generatedSchemaLinkings": "[访问次数,数据日期,1,0.5,year]"
|
||||
},
|
||||
{
|
||||
"question": "内容库近半年每个月的平均访问次数",
|
||||
"questionAugmented": "内容库近半年每个月的平均访问次数 (补充信息:。当前的日期是2023-09-04) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select MONTH(数据日期), avg(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期)",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库近半年每个月的平均访问次数 (补充信息:。当前的日期是2023-09-04) (备注: )\", we are asked:\n\"访问次数 (\" so we need column = [访问次数]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [0.5,year]. So the Schema_links are:\nSchema_links: [访问次数,数据日期,0.5,year]",
|
||||
"generatedSchemaLinkings": "[访问次数,数据日期,0.5,year]"
|
||||
},
|
||||
{
|
||||
"question": "内容库 按部门统计访问次数 top10 的部门",
|
||||
"questionAugmented": "内容库 按部门统计访问次数 top10 的部门 (补充信息:。当前的日期是2023-09-10) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 部门, sum(访问次数) from 内容库产品 group by 部门 order by sum(访问次数) desc limit 10",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"内容库 按部门统计访问次数 top10 的部门 (补充信息:。当前的日期是2023-09-10) (备注: )\", we are asked:\n\"计访问次数 \" so we need column = [访问次数]\n\" 的部门 (\" so we need column = [部门]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [10]. So the Schema_links are:\nSchema_links: [访问次数,部门,10]",
|
||||
"generatedSchemaLinkings": "[访问次数,部门,10]"
|
||||
},
|
||||
{
|
||||
"question": "超音速 近7个月，月度总访问量超过 2万的月份",
|
||||
"questionAugmented": "超音速 近7个月，月度总访问量超过 2万的月份 (补充信息:。当前的日期是2023-09-10) (备注: )",
|
||||
"dbSchema": "Table: 内容库产品, Columns = [\"用户名\", \"部门\", \"模块\", \"访问时长\", \"访问次数\", \"访问人数\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select MONTH(数据日期) from 内容库产品 where datediff('month', 数据日期, '2023-09-10') <= 7 group by MONTH(数据日期) having sum(访问次数) > 20000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"超音速 近7个月，月度总访问量超过 2万的月份 (补充信息:。当前的日期是2023-09-10) (备注: )\", we are asked:\n\"访问量超过 \" so we need column = [访问次数]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [7,20000,month]. So the Schema_links are:\nSchema_links: [访问次数,数据日期,7,20000,month]",
|
||||
"generatedSchemaLinkings": "[访问次数,数据日期,7,20000,month]"
|
||||
},
|
||||
{
|
||||
"question": "2022年7月到2023年7月之间发布到歌曲，按播放量取top 100，再按月粒度来统计近1年的运营播放量",
|
||||
"questionAugmented": "2022年7月到2023年7月之间发布到歌曲，按播放量取top 100，再按月粒度来统计近1年的运营播放量 (补充信息:。当前的日期是2023-09-10) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲语言\", \"歌曲来源\", \"运营播放量\", \"播放量\", \"歌曲名\", \"结算播放量\", \"专辑名\", \"发布日期\", \"歌曲版本\", \"歌曲类型\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select MONTH(数据日期), sum(运营播放量) from (select 数据日期, 运营播放量 from 歌曲库 where 发布日期 >= '2022-07-01' and 发布日期 <= '2023-07-01' order by 播放量 desc limit 100) t where datediff('year', 数据日期, '2023-09-10') <= 1 group by MONTH(数据日期)",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"2022年7月到2023年7月之间发布到歌曲，按播放量取top 100，再按月粒度来统计近1年的运营播放量 (补充信息:。当前的日期是2023-09-10) (备注: )\", we are asked:\n\"月之间发布到\" so we need column = [发布日期]\n\"运营播放量 \" so we need column = [播放量]\n\"运营播放量 \" so we need column = [运营播放量]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1,year,100,2022-07-01,2023-07-01]. So the Schema_links are:\nSchema_links: [发布日期,播放量,运营播放量,数据日期,1,year,100,2022-07-01,2023-07-01]",
|
||||
"generatedSchemaLinkings": "[发布日期,播放量,运营播放量,数据日期,1,year,100,2022-07-01,2023-07-01]"
|
||||
},
|
||||
{
|
||||
"question": "2022年7月到2023年7月之间发布到歌曲，按播放量取top100，再按月粒度来统计近1年的运营播放量之和，筛选出其中运营播放量之和大于2k的月份",
|
||||
"questionAugmented": "2022年7月到2023年7月之间发布到歌曲，按播放量取top100，再按月粒度来统计近1年的运营播放量之和，筛选出其中运营播放量之和大于2k的月份 (补充信息:。当前的日期是2023-09-10) (备注: )",
|
||||
"dbSchema": "Table: 歌曲库, Columns = [\"歌曲语言\", \"歌曲来源\", \"运营播放量\", \"播放量\", \"歌曲名\", \"结算播放量\", \"专辑名\", \"发布日期\", \"歌曲版本\", \"歌曲类型\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select MONTH(数据日期), sum(运营播放量) from (select 数据日期, 运营播放量 from 歌曲库 where 发布日期 >= '2022-07-01' and 发布日期 <= '2023-07-01' order by 播放量 desc limit 100) t where datediff('year', 数据日期, '2023-09-10') <= 1 group by MONTH(数据日期) having sum(运营播放量) > 2000",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"2022年7月到2023年7月之间发布到歌曲，按播放量取top100，再按月粒度来统计近1年的运营播放量之和，筛选出其中运营播放量之和大于2k的月份 (补充信息:。当前的日期是2023-09-10) (备注: )\", we are asked:\n\"月之间发布到\" so we need column = [发布日期]\n\"播放量之和，\" so we need column = [播放量]\n\"运营播放量之\" so we need column = [运营播放量]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1,2000,year,100,2022-07-01,2023-07-01]. So the Schema_links are:\nSchema_links: [发布日期,播放量,运营播放量,数据日期,1,2000,year,100,2022-07-01,2023-07-01]",
|
||||
"generatedSchemaLinkings": "[发布日期,播放量,运营播放量,数据日期,1,2000,year,100,2022-07-01,2023-07-01]"
|
||||
},
|
||||
{
|
||||
"question": "今年智能机在哪个国家的销量之和最高",
|
||||
"questionAugmented": "今年智能机在哪个国家的销量之和最高 (补充信息:’'智能机'‘是一个’机型类别‘。当前的日期是2023-11-01) (备注: )",
|
||||
"dbSchema": "Table: 营销月模型, Columns = [\"国家中文名\", \"机型类别\", \"销量\", \"数据日期\"]\nForeign_keys: []",
|
||||
"sql": "select 国家中文名, sum(销量) from 营销月模型 where 机型类别 = '智能机' and 数据日期 >= '2023-01-01' and 数据日期 <= '2023-11-01' group by 国家中文名 order by sum(销量) desc limit 1",
|
||||
"generatedSchemaLinkingCoT": "Let’s think step by step. In the question \"今年智能机在哪个国家的销量之和最高 (补充信息:’'智能机'‘是一个’机型类别‘。当前的日期是2023-11-01) (备注: )\", we are asked:\n\"国家的销量之和\" so we need column = [国家中文名]\n\"个国家的销量\" so we need column = [销量]\n\"’机型类别‘\" so we need column = [机型类别]\n\"当前的日期是\" so we need column = [数据日期]\nBased on the tables, columns, and Foreign_keys, The set of possible cell values are = [1,2023-11-01,智能机,2023-01-01]. So the Schema_links are:\nSchema_links: [国家中文名,销量,机型类别,数据日期,1,2023-11-01,智能机,2023-01-01]",
|
||||
"generatedSchemaLinkings": "[国家中文名,销量,机型类别,数据日期,1,2023-11-01,智能机,2023-01-01]"
|
||||
}
|
||||
]
|
||||
11
chat/python/instances/text2vec_instance.py
Normal file
11
chat/python/instances/text2vec_instance.py
Normal file
@@ -0,0 +1,11 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from utils.text2vec import Text2VecEmbeddingFunction
|
||||
|
||||
emb_func = Text2VecEmbeddingFunction()
|
||||
@@ -6,3 +6,4 @@ tiktoken==0.3.3
|
||||
uvicorn[standard]==0.21.1
|
||||
pandas==1.5.3
|
||||
loguru==0.7.2
|
||||
sqlglot==19.5.1
|
||||
@@ -22,7 +22,7 @@ from utils.chromadb_utils import (get_chroma_collection_size, query_chroma_colle
|
||||
add_chroma_collection, update_chroma_collection, delete_chroma_collection_by_ids,
|
||||
empty_chroma_collection_2)
|
||||
|
||||
from instances.text2vec import Text2VecEmbeddingFunction
|
||||
from utils.text2vec import Text2VecEmbeddingFunction
|
||||
|
||||
class ChromaCollectionRetriever(object):
|
||||
def __init__(self, collection:Collection):
|
||||
|
||||
@@ -14,7 +14,7 @@ import chromadb
|
||||
from chromadb.config import Settings
|
||||
from chromadb.api import Collection, Documents, Embeddings
|
||||
|
||||
from instances.text2vec import Text2VecEmbeddingFunction
|
||||
from utils.text2vec import Text2VecEmbeddingFunction
|
||||
from instances.chromadb_instance import client
|
||||
|
||||
from config.config_parse import SOLVED_QUERY_COLLECTION_NAME, PRESET_QUERY_COLLECTION_NAME
|
||||
|
||||
167
chat/python/services/s2ql/auto_cot.py
Normal file
167
chat/python/services/s2ql/auto_cot.py
Normal file
@@ -0,0 +1,167 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from typing import Any, List, Mapping, Optional, Union, Tuple
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from instances.logging_instance import logger
|
||||
from instances.text2vec_instance import emb_func
|
||||
|
||||
from sqlglot import parse_one, exp
|
||||
import numpy as np
|
||||
|
||||
def sql2schema_linking(sql: str):
|
||||
sql_ast = parse_one(sql)
|
||||
|
||||
fields_raw = []
|
||||
table_alias_map = dict()
|
||||
|
||||
literals = []
|
||||
fields = []
|
||||
|
||||
for literal in sql_ast.find_all(exp.Literal):
|
||||
literals.append(literal.output_name)
|
||||
|
||||
for column in sql_ast.find_all(exp.Column):
|
||||
fields_raw.append({
|
||||
'column_table_alias': column.table,
|
||||
'column_name': column.name,
|
||||
})
|
||||
|
||||
for table in sql_ast.find_all(exp.Table):
|
||||
if table.alias not in table_alias_map:
|
||||
table_alias_map[table.alias] = table.name
|
||||
|
||||
logger.debug(f'literals: {literals}')
|
||||
logger.debug(f'fields_raw: {fields_raw}')
|
||||
logger.debug(f'table_alias_map: {table_alias_map}')
|
||||
|
||||
for field in fields_raw:
|
||||
column_table_alias = field['column_table_alias']
|
||||
column_name = field['column_name']
|
||||
|
||||
if column_table_alias.strip() == '':
|
||||
column_table = ''
|
||||
fields.append((column_table, column_name))
|
||||
elif column_table_alias in table_alias_map:
|
||||
column_table = table_alias_map[column_table_alias]
|
||||
fields.append((column_table, column_name))
|
||||
elif column_table_alias in table_alias_map.values():
|
||||
column_table = column_table_alias
|
||||
fields.append((column_table, column_name))
|
||||
else:
|
||||
logger.error(f'column_table_alias: {column_table_alias} not in table_alias_map: {table_alias_map}')
|
||||
raise Exception(f'column_table_alias: {column_table_alias} not in table_alias_map: {table_alias_map}')
|
||||
|
||||
return {
|
||||
'fields': list(set(fields)),
|
||||
'literals': literals
|
||||
}
|
||||
|
||||
|
||||
def get_question_slices(question: str, min_window_size: int, max_window_size: int):
|
||||
assert min_window_size <= max_window_size
|
||||
assert min_window_size > 1
|
||||
assert max_window_size < len(question)+1
|
||||
|
||||
question_slices = []
|
||||
for i in range(len(question)):
|
||||
for j in range(i+1, len(question)+1):
|
||||
if j-i >= min_window_size and j-i <= max_window_size:
|
||||
question_slices.append(question[i:j])
|
||||
|
||||
return question_slices
|
||||
|
||||
|
||||
def schema_linking_match(fields: List[Tuple[str,str]], question: str, min_window_size: int, max_window_size: int):
|
||||
question_slices = get_question_slices(question, min_window_size, max_window_size)
|
||||
assert len(question_slices) > 0
|
||||
logger.debug('question_slices_len:{}'.format(len(question_slices)))
|
||||
logger.debug(f'question_slices: {question_slices}')
|
||||
|
||||
question_slices_embeddings = emb_func(question_slices)
|
||||
fields_embeddings = emb_func([field[1] for field in fields])
|
||||
|
||||
fields_embeddings = np.array(fields_embeddings) # (n_fields, 768)
|
||||
question_slices_embeddings = np.array(question_slices_embeddings) # (n_question_slices, 768)
|
||||
|
||||
question_slices_embeddings_norm = question_slices_embeddings / np.linalg.norm(question_slices_embeddings, axis=1, keepdims=True) # (n_question_slices, 768)
|
||||
question_slices_embeddings_norm_transpose = question_slices_embeddings_norm.T # (768, n_question_slices)
|
||||
|
||||
if len(fields) > 0:
|
||||
fields_embeddings_norm = fields_embeddings / np.linalg.norm(fields_embeddings, axis=1, keepdims=True) # (n_fields, 768)
|
||||
fields_question_slices_similarity = np.matmul(fields_embeddings_norm, question_slices_embeddings_norm_transpose) # (n_fields, n_question_slices)
|
||||
logger.debug('fields_question_slices_similarity_max:{}'.format(np.max(fields_question_slices_similarity, axis=1)))
|
||||
fields_question_slices_argmax = np.argmax(fields_question_slices_similarity, axis=1) # (n_fields, )
|
||||
logger.debug('fields_question_slices_argmax:{}'.format(fields_question_slices_argmax))
|
||||
|
||||
fields_question_slices_pair = []
|
||||
for i in range(len(fields)):
|
||||
if fields[i][0]!="":
|
||||
fields_question_slices_pair.append((fields[i][0]+'.'+fields[i][1], question_slices[fields_question_slices_argmax[i]]))
|
||||
else:
|
||||
fields_question_slices_pair.append((fields[i][1], question_slices[fields_question_slices_argmax[i]]))
|
||||
|
||||
logger.debug(f'fields_question_slices_pair: {fields_question_slices_pair}')
|
||||
else:
|
||||
fields_question_slices_pair = []
|
||||
|
||||
return fields_question_slices_pair
|
||||
|
||||
|
||||
def construct_schema_linking_cot(question:str, fields_question_slices_pair:List[Tuple[str,str]], literals_list:List[str]):
|
||||
cot_intro= """Let’s think step by step. In the question "{question}", we are asked:""".format(question=question)
|
||||
|
||||
schema_linkings_list = []
|
||||
|
||||
fields_cot_template = """"{question_slice}" so we need column = [{field}]"""
|
||||
fields_cot_list = []
|
||||
for field, question_slice in fields_question_slices_pair:
|
||||
fields_cot_list.append(fields_cot_template.format(question_slice=question_slice, field=field))
|
||||
schema_linkings_list.append(field)
|
||||
fields_cot = '\n'.join(fields_cot_list)
|
||||
|
||||
literals_cot_template = """Based on the tables, columns, and Foreign_keys, The set of possible cell values are = [{literals}]. So the Schema_links are:"""
|
||||
literals_cot = literals_cot_template.format(literals=",".join(literals_list))
|
||||
|
||||
schema_linkings_list += literals_list
|
||||
schema_linking_str = '[' + ",".join(schema_linkings_list) + ']'
|
||||
schema_linkings = 'Schema_links: '+ schema_linking_str
|
||||
|
||||
cot = """{cot_intro}""".format(cot_intro=cot_intro)
|
||||
if len(fields_cot_list) > 0:
|
||||
cot += '\n' + fields_cot
|
||||
|
||||
cot += '\n' + literals_cot
|
||||
cot += '\n' + schema_linkings
|
||||
|
||||
return cot, schema_linking_str
|
||||
|
||||
def auto_cot_run(question, sql, min_window_size, max_window_size):
|
||||
sql_entity = sql2schema_linking(sql)
|
||||
logger.debug(f'sql_entity: {sql_entity}')
|
||||
|
||||
fields = sql_entity['fields']
|
||||
literals = sql_entity['literals']
|
||||
|
||||
field_linked_pairs = schema_linking_match(fields, question, min_window_size, max_window_size)
|
||||
logger.debug(f'field_linked_pairs: {field_linked_pairs}')
|
||||
|
||||
auto_schema_linking_cot, auto_schema_linkings = construct_schema_linking_cot(question, field_linked_pairs, literals)
|
||||
logger.debug(f'auto_schema_linking_cot: {auto_schema_linking_cot}')
|
||||
logger.debug(f'auto_schema_linkings: {auto_schema_linkings}')
|
||||
|
||||
return auto_schema_linking_cot, auto_schema_linkings
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
question = "没有获得过奖项的高校有哪几所？"
|
||||
sql = "select 名称 from 高校 where 词条id not in ( select 高校id from 奖项 )"
|
||||
min_window_size = 6
|
||||
max_window_size = 10
|
||||
|
||||
generated_schema_linking_cot, generated_schema_linkings = auto_cot_run(question, sql, min_window_size, max_window_size)
|
||||
82
chat/python/services/s2ql/auto_cot_run.py
Normal file
82
chat/python/services/s2ql/auto_cot_run.py
Normal file
@@ -0,0 +1,82 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, List, Union, Mapping
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from instances.logging_instance import logger
|
||||
|
||||
from auto_cot import auto_cot_run
|
||||
|
||||
|
||||
|
||||
def transform_sql_example(question:str, current_date:str, table_name:str, field_list: Union[str, List[str]], prior_linkings: Union[str, Mapping[str,str]], prior_exts:str, sql:str=None):
|
||||
db_schema = f"Table: {table_name}, Columns = {field_list}\nForeign_keys: []"
|
||||
|
||||
prior_linkings_pairs = []
|
||||
if isinstance(prior_linkings, str):
|
||||
prior_linkings = prior_linkings.strip('[]')
|
||||
if prior_linkings.strip() == '':
|
||||
prior_linkings = []
|
||||
else:
|
||||
prior_linkings = prior_linkings.split(',')
|
||||
logger.debug(f'prior_linkings: {prior_linkings}')
|
||||
|
||||
for prior_linking in prior_linkings:
|
||||
logger.debug(f'prior_linking: {prior_linking}')
|
||||
entity_value, entity_type = prior_linking.split('->')
|
||||
entity_linking = """’{}‘是一个’{}‘""".format(entity_value, entity_type)
|
||||
prior_linkings_pairs.append(entity_linking)
|
||||
elif isinstance(prior_linkings, Mapping):
|
||||
for entity_value, entity_type in prior_linkings.items():
|
||||
entity_linking = """’{}‘是一个’{}‘""".format(entity_value, entity_type)
|
||||
prior_linkings_pairs.append(entity_linking)
|
||||
|
||||
prior_linkings_str = '，'.join(prior_linkings_pairs)
|
||||
|
||||
current_data_str = """当前的日期是{}""".format(current_date)
|
||||
|
||||
question_augmented = """{question} (补充信息:{prior_linking}。{current_date}) (备注: {prior_exts})""".format(question=question, prior_linking=prior_linkings_str, prior_exts=prior_exts, current_date=current_data_str)
|
||||
|
||||
return question_augmented, db_schema, sql
|
||||
|
||||
|
||||
def transform_sql_example_autoCoT_run(examplar_list, min_window_size, max_window_size):
|
||||
transformed_sql_examplar_list = []
|
||||
|
||||
for examplar in examplar_list:
|
||||
question = examplar['question']
|
||||
current_date = examplar['currentDate']
|
||||
table_name = examplar['tableName']
|
||||
field_list = examplar['fieldsList']
|
||||
prior_linkings = examplar['priorSchemaLinks']
|
||||
sql = examplar['sql']
|
||||
if 'priorExts' not in examplar:
|
||||
prior_exts = ''
|
||||
else:
|
||||
prior_exts = examplar['priorExts']
|
||||
|
||||
question_augmented, db_schema, sql = transform_sql_example(question=question, current_date=current_date, table_name=table_name, field_list=field_list, prior_linkings=prior_linkings, prior_exts=prior_exts, sql=sql)
|
||||
logger.debug(f'question_augmented: {question_augmented}')
|
||||
logger.debug(f'db_schema: {db_schema}')
|
||||
logger.debug(f'sql: {sql}')
|
||||
|
||||
generated_schema_linking_cot, generated_schema_linkings = auto_cot_run(question_augmented, sql, min_window_size, max_window_size)
|
||||
|
||||
transformed_sql_examplar = dict()
|
||||
transformed_sql_examplar['question'] = question
|
||||
transformed_sql_examplar['questionAugmented'] = question_augmented
|
||||
transformed_sql_examplar['dbSchema'] = db_schema
|
||||
transformed_sql_examplar['sql'] = sql
|
||||
transformed_sql_examplar['generatedSchemaLinkingCoT'] = generated_schema_linking_cot
|
||||
transformed_sql_examplar['generatedSchemaLinkings'] = generated_schema_linkings
|
||||
|
||||
logger.debug(f'transformed_sql_examplar: {transformed_sql_examplar}')
|
||||
|
||||
transformed_sql_examplar_list.append(transformed_sql_examplar)
|
||||
|
||||
return transformed_sql_examplar_list
|
||||
@@ -38,6 +38,9 @@ class FewShotPromptTemplate2(object):
|
||||
def delete_few_shot_example(self, example_ids: List[str])-> None:
|
||||
self.few_shot_retriever.delete_queries_by_ids(query_ids=example_ids)
|
||||
|
||||
def get_few_shot_example(self, example_ids: List[str]):
|
||||
return self.few_shot_retriever.get_query_by_ids(query_ids=example_ids)
|
||||
|
||||
def count_few_shot_example(self)-> int:
|
||||
return self.few_shot_retriever.get_query_size()
|
||||
|
||||
@@ -73,3 +76,4 @@ class FewShotPromptTemplate2(object):
|
||||
few_shot_example_str = self.few_shot_seperator.join(few_shot_example_str_unit_list)
|
||||
|
||||
return few_shot_example_str
|
||||
|
||||
40
chat/python/services/s2ql/examples_reload_run.py
Normal file
40
chat/python/services/s2ql/examples_reload_run.py
Normal file
@@ -0,0 +1,40 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Mapping
|
||||
|
||||
import requests
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from instances.logging_instance import logger
|
||||
from config.config_parse import (
|
||||
TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
LLMPARSER_HOST, LLMPARSER_PORT,)
|
||||
from few_shot_example.s2ql_examplar import examplars as sql_examplars
|
||||
|
||||
|
||||
def text2dsl_agent_wrapper_setting_update(llm_host:str, llm_port:str,
|
||||
sql_examplars:List[Mapping[str, str]],
|
||||
example_nums:int, fewshot_nums:int, self_consistency_nums:int):
|
||||
|
||||
sql_ids = [str(i) for i in range(0, len(sql_examplars))]
|
||||
|
||||
url = f"http://{llm_host}:{llm_port}/query2sql_setting_update"
|
||||
payload = {
|
||||
"sqlExamplars":sql_examplars, "sqlIds": sql_ids,
|
||||
"exampleNums":example_nums, "fewshotNums":fewshot_nums, "selfConsistencyNums":self_consistency_nums
|
||||
}
|
||||
headers = {'content-type': 'application/json'}
|
||||
response = requests.post(url, data=json.dumps(payload), headers=headers)
|
||||
logger.info(response.text)
|
||||
|
||||
if __name__ == "__main__":
|
||||
text2dsl_agent_wrapper_setting_update(LLMPARSER_HOST,LLMPARSER_PORT,
|
||||
sql_examplars, TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM)
|
||||
|
||||
|
||||
@@ -11,10 +11,10 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
from instances.logging_instance import logger
|
||||
|
||||
|
||||
def schema_link_parse(schema_link_output):
|
||||
def schema_link_parse(schema_link_output: str):
|
||||
try:
|
||||
schema_link_output = schema_link_output.strip()
|
||||
pattern = r"Schema_links:(.*)"
|
||||
pattern = r'Schema_links:(.*)'
|
||||
schema_link_output = re.findall(pattern, schema_link_output, re.DOTALL)[0].strip()
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
@@ -22,28 +22,29 @@ def schema_link_parse(schema_link_output):
|
||||
|
||||
return schema_link_output
|
||||
|
||||
|
||||
def combo_schema_link_parse(schema_linking_sql_combo_output: str):
|
||||
try:
|
||||
schema_linking_sql_combo_output = schema_linking_sql_combo_output.strip()
|
||||
pattern = r"Schema_links:(\[.*?\])"
|
||||
pattern = r'Schema_links:(\[.*?\])|Schema_links: (\[.*?\])'
|
||||
schema_links_match = re.search(pattern, schema_linking_sql_combo_output)
|
||||
|
||||
if schema_links_match:
|
||||
if schema_links_match.group(1):
|
||||
schema_links = schema_links_match.group(1)
|
||||
elif schema_links_match.group(2):
|
||||
schema_links = schema_links_match.group(2)
|
||||
else:
|
||||
schema_links = None
|
||||
|
||||
except Exception as e:
|
||||
logger.info(e)
|
||||
logger.exception(e)
|
||||
schema_links = None
|
||||
|
||||
return schema_links
|
||||
|
||||
|
||||
def combo_sql_parse(schema_linking_sql_combo_output: str):
|
||||
try:
|
||||
schema_linking_sql_combo_output = schema_linking_sql_combo_output.strip()
|
||||
pattern = r"SQL:(.*)"
|
||||
pattern = r'SQL:(.*)'
|
||||
sql_match = re.search(pattern, schema_linking_sql_combo_output)
|
||||
|
||||
if sql_match:
|
||||
@@ -55,3 +56,4 @@ def combo_sql_parse(schema_linking_sql_combo_output: str):
|
||||
sql = None
|
||||
|
||||
return sql
|
||||
|
||||
63
chat/python/services/s2ql/run.py
Normal file
63
chat/python/services/s2ql/run.py
Normal file
@@ -0,0 +1,63 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
import asyncio
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import json
|
||||
|
||||
from s2ql.constructor import FewShotPromptTemplate2
|
||||
from s2ql.sql_agent import Text2DSLAgent, Text2DSLAgentAutoCoT, Text2DSLAgentWrapper
|
||||
|
||||
from instances.llm_instance import llm
|
||||
from instances.chromadb_instance import client as chromadb_client
|
||||
from instances.logging_instance import logger
|
||||
from instances.text2vec_instance import emb_func
|
||||
|
||||
from few_shot_example.s2ql_examplar import examplars as sql_examplars
|
||||
from config.config_parse import (TEXT2DSLAGENT_COLLECTION_NAME, TEXT2DSLAGENTACT_COLLECTION_NAME,
|
||||
TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
ACT_MIN_WINDOWN_SIZE, ACT_MAX_WINDOWN_SIZE)
|
||||
|
||||
|
||||
text2dsl_agent_collection = chromadb_client.get_or_create_collection(name=TEXT2DSLAGENT_COLLECTION_NAME,
|
||||
embedding_function=emb_func,
|
||||
metadata={"hnsw:space": "cosine"})
|
||||
text2dsl_agent_act_collection = chromadb_client.get_or_create_collection(name=TEXT2DSLAGENTACT_COLLECTION_NAME,
|
||||
embedding_function=emb_func,
|
||||
metadata={"hnsw:space": "cosine"})
|
||||
|
||||
text2dsl_agent_example_prompter = FewShotPromptTemplate2(collection=text2dsl_agent_collection,
|
||||
retrieval_key="question",
|
||||
few_shot_seperator='\n\n')
|
||||
text2dsl_agent_act_example_prompter = FewShotPromptTemplate2(collection=text2dsl_agent_act_collection,
|
||||
retrieval_key="question",
|
||||
few_shot_seperator='\n\n')
|
||||
|
||||
text2sql_agent = Text2DSLAgent(num_fewshots=TEXT2DSL_FEWSHOTS_NUM, num_examples=TEXT2DSL_EXAMPLE_NUM, num_self_consistency=TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
sql_example_prompter=text2dsl_agent_example_prompter, llm=llm)
|
||||
text2sql_agent_autoCoT = Text2DSLAgentAutoCoT(num_fewshots=TEXT2DSL_FEWSHOTS_NUM, num_examples=TEXT2DSL_EXAMPLE_NUM, num_self_consistency=TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
sql_example_prompter=text2dsl_agent_act_example_prompter, llm=llm,
|
||||
auto_cot_min_window_size=ACT_MIN_WINDOWN_SIZE, auto_cot_max_window_size=ACT_MAX_WINDOWN_SIZE)
|
||||
|
||||
sql_ids = [str(i) for i in range(0, len(sql_examplars))]
|
||||
text2sql_agent.reload_setting(sql_ids, sql_examplars, TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM)
|
||||
|
||||
if text2sql_agent_autoCoT.count_examples()==0:
|
||||
source_dir_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
example_dir_path = os.path.join(source_dir_path, 'few_shot_example')
|
||||
example_json_file = os.path.join(example_dir_path, 's2ql_examplar3_transformed.json')
|
||||
with open(example_json_file, 'r') as f:
|
||||
transformed_sql_examplar_list = json.load(f)
|
||||
|
||||
transformed_sql_examplar_ids = [str(i) for i in range(0, len(transformed_sql_examplar_list))]
|
||||
text2sql_agent_autoCoT.reload_setting_autoCoT(transformed_sql_examplar_ids, transformed_sql_examplar_list, TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM)
|
||||
|
||||
|
||||
text2sql_agent_router = Text2DSLAgentWrapper(sql_agent_act=text2sql_agent_autoCoT)
|
||||
|
||||
778
chat/python/services/s2ql/sql_agent.py
Normal file
778
chat/python/services/s2ql/sql_agent.py
Normal file
@@ -0,0 +1,778 @@
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Union, Mapping, Any
|
||||
from collections import Counter
|
||||
import random
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
|
||||
from langchain.llms.base import BaseLLM
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from instances.logging_instance import logger
|
||||
|
||||
from s2ql.constructor import FewShotPromptTemplate2
|
||||
from s2ql.output_parser import schema_link_parse, combo_schema_link_parse, combo_sql_parse
|
||||
from s2ql.auto_cot_run import transform_sql_example, transform_sql_example_autoCoT_run
|
||||
|
||||
|
||||
class Text2DSLAgentBase(object):
|
||||
def __init__(self, num_fewshots:int, num_examples:int, num_self_consistency:int,
|
||||
sql_example_prompter:FewShotPromptTemplate2, llm: BaseLLM) -> None:
|
||||
self.num_fewshots = num_fewshots
|
||||
self.num_examples = num_examples
|
||||
assert self.num_fewshots <= self.num_examples
|
||||
self.num_self_consistency = num_self_consistency
|
||||
|
||||
self.llm = llm
|
||||
self.sql_example_prompter = sql_example_prompter
|
||||
|
||||
def get_examples_candidates(self, question: str, filter_condition: Mapping[str, str], num_examples: int)->List[Mapping[str, str]]:
|
||||
few_shot_example_meta_list = self.sql_example_prompter.retrieve_few_shot_example(question, num_examples, filter_condition)
|
||||
|
||||
return few_shot_example_meta_list
|
||||
|
||||
def get_fewshot_example_combos(self, example_meta_list:List[Mapping[str, str]], num_fewshots:int)-> List[List[Mapping[str, str]]]:
|
||||
fewshot_example_list = []
|
||||
for i in range(0, self.num_self_consistency):
|
||||
random.shuffle(example_meta_list)
|
||||
fewshot_example_list.append(example_meta_list[:num_fewshots])
|
||||
|
||||
return fewshot_example_list
|
||||
|
||||
def self_consistency_vote(self, output_res_pool:List[str]):
|
||||
output_res_counts = Counter(output_res_pool)
|
||||
output_res_max = output_res_counts.most_common(1)[0][0]
|
||||
total_output_num = len(output_res_pool)
|
||||
|
||||
vote_percentage = {k: (v/total_output_num) for k,v in output_res_counts.items()}
|
||||
|
||||
return output_res_max, vote_percentage
|
||||
|
||||
def schema_linking_list_str_unify(self, schema_linking_list: List[str])-> List[str]:
|
||||
schema_linking_list_unify = []
|
||||
for schema_linking_str in schema_linking_list:
|
||||
schema_linking_str_unify = ','.join(sorted([item.strip() for item in schema_linking_str.strip('[]').split(',')]))
|
||||
schema_linking_str_unify = f'[{schema_linking_str_unify}]'
|
||||
schema_linking_list_unify.append(schema_linking_str_unify)
|
||||
|
||||
return schema_linking_list_unify
|
||||
|
||||
class Text2DSLAgentAutoCoT(Text2DSLAgentBase):
|
||||
def __init__(self, num_fewshots:int, num_examples:int, num_self_consistency:int,
|
||||
sql_example_prompter:FewShotPromptTemplate2, llm: BaseLLM,
|
||||
auto_cot_min_window_size: int, auto_cot_max_window_size: int):
|
||||
super().__init__(num_fewshots, num_examples, num_self_consistency, sql_example_prompter, llm)
|
||||
|
||||
assert auto_cot_min_window_size <= auto_cot_max_window_size
|
||||
self.auto_cot_min_window_size = auto_cot_min_window_size
|
||||
self.auto_cot_max_window_size = auto_cot_max_window_size
|
||||
|
||||
def reload_setting(self, sql_example_ids: List[str], sql_example_units: List[Mapping[str,str]], num_examples:int, num_fewshots:int, num_self_consistency:int):
|
||||
self.num_fewshots = num_fewshots
|
||||
self.num_examples = num_examples
|
||||
assert self.num_fewshots <= self.num_examples
|
||||
self.num_self_consistency = num_self_consistency
|
||||
assert self.num_self_consistency >= 1
|
||||
|
||||
new_sql_example_unit_list = transform_sql_example_autoCoT_run(sql_example_units, self.auto_cot_min_window_size, self.auto_cot_max_window_size)
|
||||
self.sql_example_prompter.reload_few_shot_example(sql_example_ids, new_sql_example_unit_list)
|
||||
|
||||
def reload_setting_autoCoT(self, sql_example_ids: List[str], auto_cot_sql_example_units: List[Mapping[str,str]], num_examples:int, num_fewshots:int, num_self_consistency:int):
|
||||
self.num_fewshots = num_fewshots
|
||||
self.num_examples = num_examples
|
||||
assert self.num_fewshots <= self.num_examples
|
||||
self.num_self_consistency = num_self_consistency
|
||||
assert self.num_self_consistency >= 1
|
||||
|
||||
self.sql_example_prompter.reload_few_shot_example(sql_example_ids, auto_cot_sql_example_units)
|
||||
|
||||
def add_examples(self, sql_example_ids: List[str], sql_example_units: List[Mapping[str,str]]):
|
||||
new_sql_example_unit_list = transform_sql_example_autoCoT_run(sql_example_units, self.auto_cot_min_window_size, self.auto_cot_max_window_size)
|
||||
self.sql_example_prompter.add_few_shot_example(sql_example_ids, new_sql_example_unit_list)
|
||||
|
||||
def update_examples(self, sql_example_ids: List[str], sql_example_units: List[Mapping[str,str]]):
|
||||
new_sql_example_unit_list = transform_sql_example_autoCoT_run(sql_example_units, self.auto_cot_min_window_size, self.auto_cot_max_window_size)
|
||||
self.sql_example_prompter.update_few_shot_example(sql_example_ids, new_sql_example_unit_list)
|
||||
|
||||
def delete_examples(self, sql_example_ids: List[str]):
|
||||
self.sql_example_prompter.delete_few_shot_example(sql_example_ids)
|
||||
|
||||
def count_examples(self):
|
||||
return self.sql_example_prompter.count_few_shot_example()
|
||||
|
||||
def get_examples(self, sql_example_ids: List[str]):
|
||||
return self.sql_example_prompter.get_few_shot_example(sql_example_ids)
|
||||
|
||||
def generate_schema_linking_prompt(self, question: str, current_date:str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], prior_exts:str, fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
|
||||
instruction = "# Find the schema_links for generating SQL queries for each question based on the database schema and Foreign keys."
|
||||
|
||||
schema_linking_example_keys = ["questionAugmented", "dbSchema", "generatedSchemaLinkingCoT"]
|
||||
schema_linking_example_template = "{dbSchema}\nQ: {questionAugmented}\nA: {generatedSchemaLinkingCoT}"
|
||||
schema_linking_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=schema_linking_example_template,
|
||||
example_keys=schema_linking_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
question_augmented, db_schema, _ = transform_sql_example(question, current_date, domain_name, fields_list, prior_schema_links, prior_exts)
|
||||
new_case_template = """{dbSchema}\nQ: {questionAugmented1}\nA: Let’s think step by step. In the question "{questionAugmented2}", we are asked:"""
|
||||
new_case_prompt = new_case_template.format(dbSchema=db_schema, questionAugmented1=question_augmented, questionAugmented2=question_augmented)
|
||||
|
||||
schema_linking_prompt = instruction + '\n\n' + schema_linking_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return schema_linking_prompt
|
||||
|
||||
|
||||
def generate_schema_linking_prompt_pool(self, question: str, current_date:str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], prior_exts:str, fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
schema_linking_prompt_pool = []
|
||||
for fewshot_example_list in fewshot_example_list_pool:
|
||||
schema_linking_prompt = self.generate_schema_linking_prompt(question, current_date, domain_name, fields_list, prior_schema_links, prior_exts, fewshot_example_list)
|
||||
schema_linking_prompt_pool.append(schema_linking_prompt)
|
||||
|
||||
return schema_linking_prompt_pool
|
||||
|
||||
def generate_sql_prompt(self, question: str, domain_name: str,fields_list: List[str],
|
||||
schema_link_str: str, current_date: str, prior_schema_links: Mapping[str,str], prior_exts:str,
|
||||
fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
|
||||
instruction = "# Use the the schema links to generate the SQL queries for each of the questions."
|
||||
sql_example_keys = ["questionAugmented", "dbSchema", "generatedSchemaLinkings", "sql"]
|
||||
sql_example_template = "{dbSchema}\nQ: {questionAugmented}\nSchema_links: {generatedSchemaLinkings}\nSQL: {sql}"
|
||||
|
||||
sql_example_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=sql_example_template,
|
||||
example_keys=sql_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
question_augmented, db_schema, _ = transform_sql_example(question, current_date, domain_name, fields_list, prior_schema_links, prior_exts)
|
||||
new_case_template = "{dbSchema}\nQ: {questionAugmented}\nSchema_links: {schemaLinkings}\nSQL: "
|
||||
new_case_prompt = new_case_template.format(dbSchema=db_schema, questionAugmented=question_augmented, schemaLinkings=schema_link_str)
|
||||
|
||||
sql_example_prompt = instruction + '\n\n' + sql_example_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return sql_example_prompt
|
||||
|
||||
def generate_sql_prompt_pool(self, question: str, domain_name: str,fields_list: List[str],
|
||||
schema_link_str_pool: List[str], current_date: str, prior_schema_links: Mapping[str,str], prior_exts:str,
|
||||
fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
sql_prompt_pool = []
|
||||
for schema_link_str, fewshot_example_list in zip(schema_link_str_pool, fewshot_example_list_pool):
|
||||
sql_prompt = self.generate_sql_prompt(question, domain_name, fields_list, schema_link_str, current_date, prior_schema_links, prior_exts, fewshot_example_list)
|
||||
sql_prompt_pool.append(sql_prompt)
|
||||
|
||||
return sql_prompt_pool
|
||||
|
||||
def generate_schema_linking_sql_prompt(self, question: str, current_date:str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], prior_exts:str, fewshot_example_list:List[Mapping[str, str]]):
|
||||
|
||||
instruction = "# Find the schema_links for generating SQL queries for each question based on the database schema and Foreign keys. Then use the the schema links to generate the SQL queries for each of the questions."
|
||||
|
||||
example_keys = ["questionAugmented", "dbSchema", "generatedSchemaLinkingCoT","sql"]
|
||||
example_template = "{dbSchema}\nQ: {questionAugmented}\nA: {generatedSchemaLinkingCoT}\nSQL: {sql}"
|
||||
fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=example_template,
|
||||
example_keys=example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
question_augmented, db_schema, _ = transform_sql_example(question, current_date, domain_name, fields_list, prior_schema_links, prior_exts)
|
||||
new_case_template = """{dbSchema}\nQ: {questionAugmented1}\nA: Let’s think step by step. In the question "{questionAugmented2}", we are asked:"""
|
||||
new_case_prompt = new_case_template.format(dbSchema=db_schema, questionAugmented1=question_augmented, questionAugmented2=question_augmented)
|
||||
|
||||
prompt = instruction + '\n\n' + fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return prompt
|
||||
|
||||
def generate_schema_linking_sql_prompt_pool(self, question: str, current_date:str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], prior_exts:str, fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
schema_linking_sql_prompt_pool = []
|
||||
for fewshot_example_list in fewshot_example_list_pool:
|
||||
schema_linking_sql_prompt = self.generate_schema_linking_sql_prompt(question, current_date, domain_name, fields_list, prior_schema_links, prior_exts, fewshot_example_list)
|
||||
schema_linking_sql_prompt_pool.append(schema_linking_sql_prompt)
|
||||
|
||||
return schema_linking_sql_prompt_pool
|
||||
|
||||
async def async_query2sql(self, question: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("filter_condition: {}".format(filter_condition))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(current_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
schema_linking_prompt = self.generate_schema_linking_prompt(question, current_date, model_name, fields_list, prior_schema_links, prior_exts, fewshot_example_meta_list)
|
||||
logger.debug("schema_linking_prompt->{}".format(schema_linking_prompt))
|
||||
schema_link_output = await self.llm._call_async(schema_linking_prompt)
|
||||
logger.debug("schema_link_output->{}".format(schema_link_output))
|
||||
|
||||
schema_link_str = schema_link_parse(schema_link_output)
|
||||
logger.debug("schema_link_str->{}".format(schema_link_str))
|
||||
|
||||
sql_prompt = self.generate_sql_prompt(question, model_name, fields_list, schema_link_str, current_date, prior_schema_links, prior_exts, fewshot_example_meta_list)
|
||||
logger.debug("sql_prompt->{}".format(sql_prompt))
|
||||
sql_output = await self.llm._call_async(sql_prompt)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['currentDate'] = current_date
|
||||
|
||||
resp['schemaLinkingOutput'] = schema_link_output
|
||||
resp['schemaLinkStr'] = schema_link_str
|
||||
|
||||
resp['sqlOutput'] = sql_output
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def async_query2sql_shortcut(self, question: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("filter_condition: {}".format(filter_condition))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(current_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
schema_linking_sql_shortcut_prompt = self.generate_schema_linking_sql_prompt(question, current_date, model_name, fields_list, prior_schema_links, prior_exts, fewshot_example_meta_list)
|
||||
logger.debug("schema_linking_sql_shortcut_prompt->{}".format(schema_linking_sql_shortcut_prompt))
|
||||
schema_linking_sql_shortcut_output = await self.llm._call_async(schema_linking_sql_shortcut_prompt)
|
||||
logger.debug("schema_linking_sql_shortcut_output->{}".format(schema_linking_sql_shortcut_output))
|
||||
|
||||
schema_linking_str = combo_schema_link_parse(schema_linking_sql_shortcut_output)
|
||||
sql_str = combo_sql_parse(schema_linking_sql_shortcut_output)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['currentDate'] = current_date
|
||||
|
||||
resp['schemaLinkingComboOutput'] = schema_linking_sql_shortcut_output
|
||||
resp['schemaLinkStr'] = schema_linking_str
|
||||
resp['sqlOutput'] = sql_str
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def generate_schema_linking_tasks(self, question: str, model_name: str, fields_list: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str, fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
|
||||
schema_linking_prompt_pool = self.generate_schema_linking_prompt_pool(question, current_date, model_name, fields_list, prior_schema_links, prior_exts, fewshot_example_list_combo)
|
||||
logger.debug("schema_linking_prompt_pool->{}".format(schema_linking_prompt_pool))
|
||||
schema_linking_output_pool = await asyncio.gather(*[self.llm._call_async(schema_linking_prompt) for schema_linking_prompt in schema_linking_prompt_pool])
|
||||
logger.debug("schema_linking_output_pool->{}".format(schema_linking_output_pool))
|
||||
|
||||
schema_linking_str_pool = [schema_link_parse(schema_linking_output) for schema_linking_output in schema_linking_output_pool]
|
||||
|
||||
return schema_linking_str_pool
|
||||
|
||||
async def generate_sql_tasks(self, question: str, model_name: str, fields_list: List[str], schema_link_str_pool: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str, fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
|
||||
sql_prompt_pool = self.generate_sql_prompt_pool(question, model_name, fields_list, schema_link_str_pool, current_date, prior_schema_links, prior_exts, fewshot_example_list_combo)
|
||||
logger.debug("sql_prompt_pool->{}".format(sql_prompt_pool))
|
||||
sql_output_pool = await asyncio.gather(*[self.llm._call_async(sql_prompt) for sql_prompt in sql_prompt_pool])
|
||||
logger.debug("sql_output_pool->{}".format(sql_output_pool))
|
||||
|
||||
return sql_output_pool
|
||||
|
||||
async def generate_schema_linking_sql_tasks(self, question: str, model_name: str, fields_list: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str, fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
schema_linking_sql_prompt_pool = self.generate_schema_linking_sql_prompt_pool(question, current_date, model_name, fields_list, prior_schema_links, prior_exts, fewshot_example_list_combo)
|
||||
schema_linking_sql_output_task_pool = [self.llm._call_async(schema_linking_sql_prompt) for schema_linking_sql_prompt in schema_linking_sql_prompt_pool]
|
||||
schema_linking_sql_output_res_pool = await asyncio.gather(*schema_linking_sql_output_task_pool)
|
||||
logger.debug("schema_linking_sql_output_res_pool->{}".format(schema_linking_sql_output_res_pool))
|
||||
|
||||
return schema_linking_sql_output_res_pool
|
||||
|
||||
async def tasks_run(self, question: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("filter_condition: {}".format(filter_condition))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(current_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
fewshot_example_list_combo = self.get_fewshot_example_combos(fewshot_example_meta_list, self.num_fewshots)
|
||||
|
||||
schema_linking_candidate_list = await self.generate_schema_linking_tasks(question, model_name, fields_list, current_date, prior_schema_links, prior_exts, fewshot_example_list_combo)
|
||||
logger.debug(f'schema_linking_candidate_list:{schema_linking_candidate_list}')
|
||||
schema_linking_candidate_sorted_list = self.schema_linking_list_str_unify(schema_linking_candidate_list)
|
||||
logger.debug(f'schema_linking_candidate_sorted_list:{schema_linking_candidate_sorted_list}')
|
||||
|
||||
schema_linking_output_max, schema_linking_output_vote_percentage = self.self_consistency_vote(schema_linking_candidate_sorted_list)
|
||||
|
||||
sql_output_candicates = await self.generate_sql_tasks(question, model_name, fields_list, schema_linking_candidate_list, current_date, prior_schema_links, prior_exts, fewshot_example_list_combo)
|
||||
logger.debug(f'sql_output_candicates:{sql_output_candicates}')
|
||||
sql_output_max, sql_output_vote_percentage = self.self_consistency_vote(sql_output_candicates)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['currentDate'] = current_date
|
||||
|
||||
resp['schemaLinkStr'] = schema_linking_output_max
|
||||
resp['schemaLinkingWeight'] = schema_linking_output_vote_percentage
|
||||
|
||||
resp['sqlOutput'] = sql_output_max
|
||||
resp['sqlWeight'] = sql_output_vote_percentage
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def tasks_run_shortcut(self, question: str, filter_condition: Mapping[str,str], model_name: str, fields_list: List[str],
|
||||
current_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("filter_condition: {}".format(filter_condition))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(current_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
fewshot_example_list_combo = self.get_fewshot_example_combos(fewshot_example_meta_list, self.num_fewshots)
|
||||
|
||||
schema_linking_sql_output_candidates = await self.generate_schema_linking_sql_tasks(question, model_name, fields_list, current_date, prior_schema_links, prior_exts, fewshot_example_list_combo)
|
||||
logger.debug(f'schema_linking_sql_output_candidates:{schema_linking_sql_output_candidates}')
|
||||
schema_linking_output_candidate_list = [combo_schema_link_parse(schema_linking_sql_output_candidate) for schema_linking_sql_output_candidate in schema_linking_sql_output_candidates]
|
||||
logger.debug(f'schema_linking_sql_output_candidate_list:{schema_linking_output_candidate_list}')
|
||||
schema_linking_output_candidate_sorted_list = self.schema_linking_list_str_unify(schema_linking_output_candidate_list)
|
||||
|
||||
schema_linking_output_max, schema_linking_output_vote_percentage = self.self_consistency_vote(schema_linking_output_candidate_sorted_list)
|
||||
|
||||
sql_output_candidate_list = [combo_sql_parse(schema_linking_sql_output_candidate) for schema_linking_sql_output_candidate in schema_linking_sql_output_candidates]
|
||||
logger.debug(f'sql_output_candidate_list:{sql_output_candidate_list}')
|
||||
sql_output_max, sql_output_vote_percentage = self.self_consistency_vote(sql_output_candidate_list)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['currentDate'] = current_date
|
||||
|
||||
resp['schemaLinkStr'] = schema_linking_output_max
|
||||
resp['schemaLinkingWeight'] = schema_linking_output_vote_percentage
|
||||
|
||||
resp['sqlOutput'] = sql_output_max
|
||||
resp['sqlWeight'] = sql_output_vote_percentage
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
class Text2DSLAgent(Text2DSLAgentBase):
|
||||
def __init__(self, num_fewshots:int, num_examples:int, num_self_consistency:int,
|
||||
sql_example_prompter:FewShotPromptTemplate2, llm: BaseLLM,) -> None:
|
||||
super().__init__(num_fewshots, num_examples, num_self_consistency, sql_example_prompter, llm)
|
||||
|
||||
def reload_setting(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]], num_examples:int, num_fewshots:int, num_self_consistency:int):
|
||||
self.num_fewshots = num_fewshots
|
||||
self.num_examples = num_examples
|
||||
assert self.num_fewshots <= self.num_examples
|
||||
self.num_self_consistency = num_self_consistency
|
||||
assert self.num_self_consistency >= 1
|
||||
self.sql_example_prompter.reload_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def add_examples(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]]):
|
||||
self.sql_example_prompter.add_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def update_examples(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]]):
|
||||
self.sql_example_prompter.update_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def delete_examples(self, sql_example_ids:List[str]):
|
||||
self.sql_example_prompter.delete_few_shot_example(sql_example_ids)
|
||||
|
||||
def get_examples(self, sql_example_ids: List[str]):
|
||||
return self.sql_example_prompter.get_few_shot_example(sql_example_ids)
|
||||
|
||||
def count_examples(self):
|
||||
return self.sql_example_prompter.count_few_shot_example()
|
||||
|
||||
def generate_schema_linking_prompt(self, question: str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
|
||||
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
|
||||
|
||||
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links"
|
||||
|
||||
schema_linking_example_keys = ["tableName", "fieldsList", "priorSchemaLinks", "question", "analysis", "schemaLinks"]
|
||||
schema_linking_example_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schemaLinks}"
|
||||
schema_linking_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=schema_linking_example_template,
|
||||
example_keys=schema_linking_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\n问题:{question}\n分析: 让我们一步一步地思考。"
|
||||
new_case_prompt = new_case_template.format(tableName=domain_name, fieldsList=fields_list, priorSchemaLinks=prior_schema_links_str, question=question)
|
||||
|
||||
schema_linking_prompt = instruction + '\n\n' + schema_linking_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
return schema_linking_prompt
|
||||
|
||||
def generate_schema_linking_prompt_pool(self, question: str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
schema_linking_prompt_pool = []
|
||||
for fewshot_example_list in fewshot_example_list_pool:
|
||||
schema_linking_prompt = self.generate_schema_linking_prompt(question, domain_name, fields_list, prior_schema_links, fewshot_example_list)
|
||||
schema_linking_prompt_pool.append(schema_linking_prompt)
|
||||
|
||||
return schema_linking_prompt_pool
|
||||
|
||||
def generate_sql_prompt(self, question: str, domain_name: str,
|
||||
schema_link_str: str, data_date: str,
|
||||
fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
instruction = "# 根据schema_links为每个问题生成SQL查询语句"
|
||||
sql_example_keys = ["question", "currentDate", "tableName", "schemaLinks", "sql"]
|
||||
sql_example_template = "问题:{question}\nCurrent_date:{currentDate}\nTable {tableName}\nSchema_links:{schemaLinks}\nSQL:{sql}"
|
||||
|
||||
sql_example_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=sql_example_template,
|
||||
example_keys=sql_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "问题:{question}\nCurrent_date:{currentDate}\nTable {tableName}\nSchema_links:{schemaLinks}\nSQL:"
|
||||
new_case_prompt = new_case_template.format(question=question, currentDate=data_date, tableName=domain_name, schemaLinks=schema_link_str)
|
||||
|
||||
sql_example_prompt = instruction + '\n\n' + sql_example_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return sql_example_prompt
|
||||
|
||||
def generate_sql_prompt_pool(self, question: str, domain_name: str, data_date: str,
|
||||
schema_link_str_pool: List[str], fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
sql_prompt_pool = []
|
||||
for schema_link_str, fewshot_example_list in zip(schema_link_str_pool, fewshot_example_list_pool):
|
||||
sql_prompt = self.generate_sql_prompt(question, domain_name, schema_link_str, data_date, fewshot_example_list)
|
||||
sql_prompt_pool.append(sql_prompt)
|
||||
|
||||
return sql_prompt_pool
|
||||
|
||||
def generate_schema_linking_sql_prompt(self, question: str,
|
||||
domain_name: str,
|
||||
data_date : str,
|
||||
fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str],
|
||||
fewshot_example_list:List[Mapping[str, str]]):
|
||||
|
||||
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
|
||||
|
||||
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links,再根据schema_links为每个问题生成SQL查询语句"
|
||||
|
||||
example_keys = ["tableName", "fieldsList", "priorSchemaLinks", "currentDate", "question", "analysis", "schemaLinks", "sql"]
|
||||
example_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\nCurrent_date:{currentDate}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schemaLinks}\nSQL:{sql}"
|
||||
fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=example_template,
|
||||
example_keys=example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\nCurrent_date:{currentDate}\n问题:{question}\n分析: 让我们一步一步地思考。"
|
||||
new_case_prompt = new_case_template.format(tableName=domain_name, fieldsList=fields_list, priorSchemaLinks=prior_schema_links_str, currentDate=data_date, question=question)
|
||||
|
||||
prompt = instruction + '\n\n' + fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return prompt
|
||||
|
||||
def generate_schema_linking_sql_prompt_pool(self, question: str, domain_name: str, fields_list: List[str], data_date: str,
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
schema_linking_sql_prompt_pool = []
|
||||
for fewshot_example_list in fewshot_example_list_pool:
|
||||
schema_linking_sql_prompt = self.generate_schema_linking_sql_prompt(question, domain_name, data_date, fields_list, prior_schema_links, fewshot_example_list)
|
||||
schema_linking_sql_prompt_pool.append(schema_linking_sql_prompt)
|
||||
|
||||
return schema_linking_sql_prompt_pool
|
||||
|
||||
def self_consistency_vote(self, output_res_pool:List[str]):
|
||||
output_res_counts = Counter(output_res_pool)
|
||||
output_res_max = output_res_counts.most_common(1)[0][0]
|
||||
total_output_num = len(output_res_pool)
|
||||
|
||||
vote_percentage = {k: (v/total_output_num) for k,v in output_res_counts.items()}
|
||||
|
||||
return output_res_max, vote_percentage
|
||||
|
||||
def schema_linking_list_str_unify(self, schema_linking_list: List[str])-> List[str]:
|
||||
schema_linking_list_unify = []
|
||||
for schema_linking_str in schema_linking_list:
|
||||
schema_linking_str_unify = ','.join(sorted([item.strip() for item in schema_linking_str.strip('[]').split(',')]))
|
||||
schema_linking_str_unify = f'[{schema_linking_str_unify}]'
|
||||
schema_linking_list_unify.append(schema_linking_str_unify)
|
||||
|
||||
return schema_linking_list_unify
|
||||
|
||||
async def generate_schema_linking_tasks(self, question: str, domain_name: str,
|
||||
fields_list: List[str], prior_schema_links: Mapping[str,str],
|
||||
fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
|
||||
schema_linking_prompt_pool = self.generate_schema_linking_prompt_pool(question, domain_name,
|
||||
fields_list, prior_schema_links,
|
||||
fewshot_example_list_combo)
|
||||
schema_linking_output_task_pool = [self.llm._call_async(schema_linking_prompt) for schema_linking_prompt in schema_linking_prompt_pool]
|
||||
schema_linking_output_pool = await asyncio.gather(*schema_linking_output_task_pool)
|
||||
logger.debug(f'schema_linking_output_pool:{schema_linking_output_pool}')
|
||||
|
||||
schema_linking_str_pool = [schema_link_parse(schema_linking_output) for schema_linking_output in schema_linking_output_pool]
|
||||
|
||||
return schema_linking_str_pool
|
||||
|
||||
async def generate_sql_tasks(self, question: str, domain_name: str, data_date: str,
|
||||
schema_link_str_pool: List[str], fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
|
||||
sql_prompt_pool = self.generate_sql_prompt_pool(question, domain_name, schema_link_str_pool, data_date, fewshot_example_list_combo)
|
||||
sql_output_task_pool = [self.llm._call_async(sql_prompt) for sql_prompt in sql_prompt_pool]
|
||||
sql_output_res_pool = await asyncio.gather(*sql_output_task_pool)
|
||||
logger.debug(f'sql_output_res_pool:{sql_output_res_pool}')
|
||||
|
||||
return sql_output_res_pool
|
||||
|
||||
async def generate_schema_linking_sql_tasks(self, question: str, domain_name: str, fields_list: List[str], data_date: str,
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
schema_linking_sql_prompt_pool = self.generate_schema_linking_sql_prompt_pool(question, domain_name, fields_list, data_date, prior_schema_links, fewshot_example_list_combo)
|
||||
schema_linking_sql_output_task_pool = [self.llm._call_async(schema_linking_sql_prompt) for schema_linking_sql_prompt in schema_linking_sql_prompt_pool]
|
||||
schema_linking_sql_output_res_pool = await asyncio.gather(*schema_linking_sql_output_task_pool)
|
||||
logger.debug(f'schema_linking_sql_output_res_pool:{schema_linking_sql_output_res_pool}')
|
||||
|
||||
return schema_linking_sql_output_res_pool
|
||||
|
||||
async def tasks_run(self, question: str, filter_condition: Mapping[str, str], domain_name: str, fields_list: List[str], prior_schema_links: Mapping[str,str], data_date: str, prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("domain_name: {}".format(domain_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
question = question + ' 备注:'+prior_exts
|
||||
logger.info("question_prior_exts: {}".format(question))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
fewshot_example_list_combo = self.get_fewshot_example_combos(fewshot_example_meta_list, self.num_fewshots)
|
||||
|
||||
schema_linking_candidate_list = await self.generate_schema_linking_tasks(question, domain_name, fields_list, prior_schema_links, fewshot_example_list_combo)
|
||||
logger.debug(f'schema_linking_candidate_list:{schema_linking_candidate_list}')
|
||||
schema_linking_candidate_sorted_list = self.schema_linking_list_str_unify(schema_linking_candidate_list)
|
||||
logger.debug(f'schema_linking_candidate_sorted_list:{schema_linking_candidate_sorted_list}')
|
||||
|
||||
schema_linking_output_max, schema_linking_output_vote_percentage = self.self_consistency_vote(schema_linking_candidate_sorted_list)
|
||||
|
||||
sql_output_candicates = await self.generate_sql_tasks(question, domain_name, data_date, schema_linking_candidate_list,fewshot_example_list_combo)
|
||||
logger.debug(f'sql_output_candicates:{sql_output_candicates}')
|
||||
sql_output_max, sql_output_vote_percentage = self.self_consistency_vote(sql_output_candicates)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = domain_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkStr'] = schema_linking_output_max
|
||||
resp['schemaLinkingWeight'] = schema_linking_output_vote_percentage
|
||||
|
||||
resp['sqlOutput'] = sql_output_max
|
||||
resp['sqlWeight'] = sql_output_vote_percentage
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def tasks_run_shortcut(self, question: str, filter_condition: Mapping[str, str], domain_name: str, fields_list: List[str], prior_schema_links: Mapping[str,str], data_date: str, prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("domain_name: {}".format(domain_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
question = question + ' 备注:'+prior_exts
|
||||
logger.info("question_prior_exts: {}".format(question))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
fewshot_example_list_combo = self.get_fewshot_example_combos(fewshot_example_meta_list, self.num_fewshots)
|
||||
|
||||
schema_linking_sql_output_candidates = await self.generate_schema_linking_sql_tasks(question, domain_name, fields_list, data_date, prior_schema_links, fewshot_example_list_combo)
|
||||
logger.debug(f'schema_linking_sql_output_candidates:{schema_linking_sql_output_candidates}')
|
||||
schema_linking_output_candidate_list = [combo_schema_link_parse(schema_linking_sql_output_candidate) for schema_linking_sql_output_candidate in schema_linking_sql_output_candidates]
|
||||
logger.debug(f'schema_linking_sql_output_candidate_list:{schema_linking_output_candidate_list}')
|
||||
schema_linking_output_candidate_sorted_list = self.schema_linking_list_str_unify(schema_linking_output_candidate_list)
|
||||
|
||||
schema_linking_output_max, schema_linking_output_vote_percentage = self.self_consistency_vote(schema_linking_output_candidate_sorted_list)
|
||||
|
||||
sql_output_candidate_list = [combo_sql_parse(schema_linking_sql_output_candidate) for schema_linking_sql_output_candidate in schema_linking_sql_output_candidates]
|
||||
logger.debug(f'sql_output_candidate_list:{sql_output_candidate_list}')
|
||||
sql_output_max, sql_output_vote_percentage = self.self_consistency_vote(sql_output_candidate_list)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = domain_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkStr'] = schema_linking_output_max
|
||||
resp['schemaLinkingWeight'] = schema_linking_output_vote_percentage
|
||||
|
||||
resp['sqlOutput'] = sql_output_max
|
||||
resp['sqlWeight'] = sql_output_vote_percentage
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def async_query2sql(self, question: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
data_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("data_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
question = question + ' 备注:'+prior_exts
|
||||
logger.info("question_prior_exts: {}".format(question))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
schema_linking_prompt = self.generate_schema_linking_prompt(question, model_name, fields_list, prior_schema_links, fewshot_example_meta_list)
|
||||
logger.debug("schema_linking_prompt->{}".format(schema_linking_prompt))
|
||||
schema_link_output = await self.llm._call_async(schema_linking_prompt)
|
||||
|
||||
schema_link_str = schema_link_parse(schema_link_output)
|
||||
|
||||
sql_prompt = self.generate_sql_prompt(question, model_name, schema_link_str, data_date, fewshot_example_meta_list)
|
||||
logger.debug("sql_prompt->{}".format(sql_prompt))
|
||||
sql_output = await self.llm._call_async(sql_prompt)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkingOutput'] = schema_link_output
|
||||
resp['schemaLinkStr'] = schema_link_str
|
||||
|
||||
resp['sqlOutput'] = sql_output
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def async_query2sql_shortcut(self, question: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
data_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
|
||||
logger.info("question: {}".format(question))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("data_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
question = question + ' 备注:'+prior_exts
|
||||
logger.info("question_prior_exts: {}".format(question))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(question, filter_condition, self.num_examples)
|
||||
schema_linking_sql_shortcut_prompt = self.generate_schema_linking_sql_prompt(question, model_name, data_date, fields_list, prior_schema_links, fewshot_example_meta_list)
|
||||
logger.debug("schema_linking_sql_shortcut_prompt->{}".format(schema_linking_sql_shortcut_prompt))
|
||||
schema_linking_sql_shortcut_output = await self.llm._call_async(schema_linking_sql_shortcut_prompt)
|
||||
|
||||
schema_linking_str = combo_schema_link_parse(schema_linking_sql_shortcut_output)
|
||||
sql_str = combo_sql_parse(schema_linking_sql_shortcut_output)
|
||||
|
||||
resp = dict()
|
||||
resp['question'] = question
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkingComboOutput'] = schema_linking_sql_shortcut_output
|
||||
resp['schemaLinkStr'] = schema_linking_str
|
||||
resp['sqlOutput'] = sql_str
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
class SqlModeEnum(Enum):
|
||||
VALUE5 = '1_pass_auto_cot'
|
||||
VALUE6 = '1_pass_auto_cot_self_consistency'
|
||||
VALUE7 = '2_pass_auto_cot'
|
||||
VALUE8 = '2_pass_auto_cot_self_consistency'
|
||||
|
||||
class Text2DSLAgentWrapper(object):
|
||||
def __init__(self, sql_agent_act:Text2DSLAgentAutoCoT):
|
||||
self.sql_agent_act = sql_agent_act
|
||||
|
||||
async def async_query2sql(self, question: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
data_date: str, prior_schema_links: Mapping[str,str], prior_exts: str, sql_generation_mode: str):
|
||||
|
||||
if sql_generation_mode not in (sql_mode.value for sql_mode in SqlModeEnum):
|
||||
raise ValueError(f"sql_generation_mode: {sql_generation_mode} is not in SqlModeEnum")
|
||||
|
||||
if sql_generation_mode == '1_pass_auto_cot':
|
||||
logger.info(f"sql wrapper: {sql_generation_mode}")
|
||||
resp = await self.sql_agent_act.async_query2sql_shortcut(question=question, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list, current_date=data_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
return resp
|
||||
elif sql_generation_mode == '1_pass_auto_cot_self_consistency':
|
||||
logger.info(f"sql wrapper: {sql_generation_mode}")
|
||||
resp = await self.sql_agent_act.tasks_run_shortcut(question=question, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list, current_date=data_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
return resp
|
||||
elif sql_generation_mode == '2_pass_auto_cot':
|
||||
logger.info(f"sql wrapper: {sql_generation_mode}")
|
||||
resp = await self.sql_agent_act.async_query2sql(question=question, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list, current_date=data_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
return resp
|
||||
elif sql_generation_mode == '2_pass_auto_cot_self_consistency':
|
||||
logger.info(f"sql wrapper: {sql_generation_mode}")
|
||||
resp = await self.sql_agent_act.tasks_run(question=question, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list, current_date=data_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
return resp
|
||||
else:
|
||||
raise ValueError(f'sql_generation_mode:{sql_generation_mode} is not in SqlModeEnum')
|
||||
|
||||
def update_configs(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]],
|
||||
num_examples: int, num_fewshots: int, num_self_consistency: int):
|
||||
self.sql_agent_act.reload_setting(sql_example_ids=sql_example_ids, sql_example_units=sql_example_units, num_examples=num_examples, num_fewshots=num_fewshots, num_self_consistency=num_self_consistency)
|
||||
|
||||
def add_examples(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]]):
|
||||
self.sql_agent_act.add_examples(sql_example_ids=sql_example_ids, sql_example_units=sql_example_units)
|
||||
|
||||
def update_examples(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]]):
|
||||
self.sql_agent_act.update_examples(sql_example_ids=sql_example_ids, sql_example_units=sql_example_units)
|
||||
|
||||
def delete_examples(self, sql_example_ids:List[str]):
|
||||
self.sql_agent_act.delete_examples(sql_example_ids=sql_example_ids)
|
||||
|
||||
def get_examples(self, sql_example_ids: List[str]):
|
||||
sql_agent_act_examples = self.sql_agent_act.get_examples(sql_example_ids=sql_example_ids)
|
||||
|
||||
return sql_agent_act_examples
|
||||
|
||||
def count_examples(self):
|
||||
sql_agent_examples_act_cnt = self.sql_agent_act.count_examples()
|
||||
|
||||
return sql_agent_examples_act_cnt
|
||||
@@ -1,61 +0,0 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Mapping
|
||||
|
||||
import requests
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from instances.logging_instance import logger
|
||||
|
||||
from config.config_parse import (TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
LLMPARSER_HOST, LLMPARSER_PORT, TEXT2DSL_IS_SHORTCUT, TEXT2DSL_IS_SELF_CONSISTENCY)
|
||||
from few_shot_example.sql_examplar import examplars as sql_examplars
|
||||
|
||||
|
||||
def text2sql_agent_setting_update(llm_host:str, llm_port:str,
|
||||
sql_examplars:List[Mapping[str, str]], example_nums:int):
|
||||
|
||||
url = f"http://{llm_host}:{llm_port}/text2sql_agent_setting_update/"
|
||||
payload = {"sqlExamplars":sql_examplars, "exampleNums":example_nums}
|
||||
headers = {'content-type': 'application/json'}
|
||||
response = requests.post(url, data=json.dumps(payload), headers=headers)
|
||||
logger.info(response.text)
|
||||
|
||||
|
||||
def text2dsl_agent_cs_setting_update(llm_host:str, llm_port:str,
|
||||
sql_examplars:List[Mapping[str, str]], example_nums:int, fewshot_nums:int, self_consistency_nums:int):
|
||||
|
||||
url = f"http://{llm_host}:{llm_port}/texg2sqt_cs_agent_setting_update/"
|
||||
payload = {"sqlExamplars":sql_examplars,
|
||||
"exampleNums":example_nums, "fewshotNums":fewshot_nums, "selfConsistencyNums":self_consistency_nums}
|
||||
headers = {'content-type': 'application/json'}
|
||||
response = requests.post(url, data=json.dumps(payload), headers=headers)
|
||||
logger.info(response.text)
|
||||
|
||||
|
||||
def text2dsl_agent_wrapper_setting_update(llm_host:str, llm_port:str,
|
||||
is_shortcut:bool, is_self_consistency:bool,
|
||||
sql_examplars:List[Mapping[str, str]], example_nums:int, fewshot_nums:int, self_consistency_nums:int):
|
||||
|
||||
sql_ids = [str(i) for i in range(0, len(sql_examplars))]
|
||||
|
||||
url = f"http://{llm_host}:{llm_port}/query2sql_setting_update/"
|
||||
payload = {"isShortcut":is_shortcut, "isSelfConsistency":is_self_consistency,
|
||||
"sqlExamplars":sql_examplars, "sqlIds": sql_ids,
|
||||
"exampleNums":example_nums, "fewshotNums":fewshot_nums, "selfConsistencyNums":self_consistency_nums}
|
||||
headers = {'content-type': 'application/json'}
|
||||
response = requests.post(url, data=json.dumps(payload), headers=headers)
|
||||
logger.info(response.text)
|
||||
|
||||
if __name__ == "__main__":
|
||||
text2dsl_agent_wrapper_setting_update(LLMPARSER_HOST,LLMPARSER_PORT,
|
||||
TEXT2DSL_IS_SHORTCUT, TEXT2DSL_IS_SELF_CONSISTENCY,
|
||||
sql_examplars, TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM)
|
||||
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
import asyncio
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from sql.constructor import FewShotPromptTemplate2
|
||||
from sql.sql_agent import Text2DSLAgent, Text2DSLAgentConsistency, Text2DSLAgentWrapper
|
||||
|
||||
from instances.llm_instance import llm
|
||||
from instances.text2vec import Text2VecEmbeddingFunction
|
||||
from instances.chromadb_instance import client
|
||||
from instances.logging_instance import logger
|
||||
|
||||
from few_shot_example.sql_examplar import examplars as sql_examplars
|
||||
from config.config_parse import (TEXT2DSLAGENT_COLLECTION_NAME, TEXT2DSLAGENTCS_COLLECTION_NAME,
|
||||
TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
TEXT2DSL_IS_SHORTCUT, TEXT2DSL_IS_SELF_CONSISTENCY)
|
||||
|
||||
|
||||
emb_func = Text2VecEmbeddingFunction()
|
||||
text2dsl_agent_collection = client.get_or_create_collection(name=TEXT2DSLAGENT_COLLECTION_NAME,
|
||||
embedding_function=emb_func,
|
||||
metadata={"hnsw:space": "cosine"})
|
||||
text2dsl_agentcs_collection = client.get_or_create_collection(name=TEXT2DSLAGENTCS_COLLECTION_NAME,
|
||||
embedding_function=emb_func,
|
||||
metadata={"hnsw:space": "cosine"})
|
||||
|
||||
text2dsl_agent_example_prompter = FewShotPromptTemplate2(collection=text2dsl_agent_collection,
|
||||
retrieval_key="question",
|
||||
few_shot_seperator='\n\n')
|
||||
|
||||
text2dsl_agentcs_example_prompter = FewShotPromptTemplate2(collection=text2dsl_agentcs_collection,
|
||||
retrieval_key="question",
|
||||
few_shot_seperator='\n\n')
|
||||
|
||||
text2sql_agent = Text2DSLAgent(num_fewshots=TEXT2DSL_EXAMPLE_NUM,
|
||||
sql_example_prompter=text2dsl_agent_example_prompter, llm=llm)
|
||||
|
||||
text2sql_cs_agent = Text2DSLAgentConsistency(num_fewshots=TEXT2DSL_FEWSHOTS_NUM, num_examples=TEXT2DSL_EXAMPLE_NUM, num_self_consistency=TEXT2DSL_SELF_CONSISTENCY_NUM,
|
||||
sql_example_prompter=text2dsl_agentcs_example_prompter, llm=llm)
|
||||
|
||||
sql_ids = [str(i) for i in range(0, len(sql_examplars))]
|
||||
text2sql_agent.reload_setting(sql_ids, sql_examplars, TEXT2DSL_EXAMPLE_NUM)
|
||||
text2sql_cs_agent.reload_setting(sql_ids, sql_examplars, TEXT2DSL_EXAMPLE_NUM, TEXT2DSL_FEWSHOTS_NUM, TEXT2DSL_SELF_CONSISTENCY_NUM)
|
||||
|
||||
|
||||
text2sql_agent_router = Text2DSLAgentWrapper(sql_agent=text2sql_agent, sql_agent_cs=text2sql_cs_agent,
|
||||
is_shortcut=TEXT2DSL_IS_SHORTCUT, is_self_consistency=TEXT2DSL_IS_SELF_CONSISTENCY)
|
||||
@@ -1,405 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Union, Mapping, Any
|
||||
from collections import Counter
|
||||
import random
|
||||
import asyncio
|
||||
from langchain.llms.base import BaseLLM
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from instances.logging_instance import logger
|
||||
|
||||
from sql.constructor import FewShotPromptTemplate2
|
||||
from sql.output_parser import schema_link_parse, combo_schema_link_parse, combo_sql_parse
|
||||
|
||||
|
||||
class Text2DSLAgent(object):
|
||||
def __init__(self, num_fewshots:int,
|
||||
sql_example_prompter:FewShotPromptTemplate2,
|
||||
llm: BaseLLM):
|
||||
self.num_fewshots = num_fewshots
|
||||
self.sql_example_prompter = sql_example_prompter
|
||||
self.llm = llm
|
||||
|
||||
def reload_setting(self, sql_example_ids: List[str], sql_example_units: List[Mapping[str,str]], num_fewshots: int):
|
||||
self.num_fewshots = num_fewshots
|
||||
|
||||
self.sql_example_prompter.reload_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def add_examples(self, sql_example_ids: List[str], sql_example_units: List[Mapping[str,str]]):
|
||||
self.sql_example_prompter.add_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def update_examples(self, sql_example_ids: List[str], sql_example_units: List[Mapping[str,str]]):
|
||||
self.sql_example_prompter.update_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def delete_examples(self, sql_example_ids: List[str]):
|
||||
self.sql_example_prompter.delete_few_shot_example(sql_example_ids)
|
||||
|
||||
def count_examples(self):
|
||||
return self.sql_example_prompter.count_few_shot_example()
|
||||
|
||||
def get_fewshot_examples(self, query_text: str, filter_condition: Mapping[str,str])->List[Mapping[str, str]]:
|
||||
few_shot_example_meta_list = self.sql_example_prompter.retrieve_few_shot_example(query_text, self.num_fewshots, filter_condition)
|
||||
|
||||
return few_shot_example_meta_list
|
||||
|
||||
def generate_schema_linking_prompt(self, user_query: str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
|
||||
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
|
||||
|
||||
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links"
|
||||
|
||||
schema_linking_example_keys = ["tableName", "fieldsList", "priorSchemaLinks", "question", "analysis", "schemaLinks"]
|
||||
schema_linking_example_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schemaLinks}"
|
||||
schema_linking_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=schema_linking_example_template,
|
||||
example_keys=schema_linking_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\n问题:{question}\n分析: 让我们一步一步地思考。"
|
||||
new_case_prompt = new_case_template.format(tableName=domain_name, fieldsList=fields_list, priorSchemaLinks=prior_schema_links_str, question=user_query)
|
||||
|
||||
schema_linking_prompt = instruction + '\n\n' + schema_linking_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
return schema_linking_prompt
|
||||
|
||||
def generate_sql_prompt(self, user_query: str, domain_name: str,
|
||||
schema_link_str: str, data_date: str,
|
||||
fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
instruction = "# 根据schema_links为每个问题生成SQL查询语句"
|
||||
sql_example_keys = ["question", "currentDate", "tableName", "schemaLinks", "sql"]
|
||||
sql_example_template = "问题:{question}\nCurrent_date:{currentDate}\nTable {tableName}\nSchema_links:{schemaLinks}\nSQL:{sql}"
|
||||
|
||||
sql_example_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=sql_example_template,
|
||||
example_keys=sql_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "问题:{question}\nCurrent_date:{currentDate}\nTable {tableName}\nSchema_links:{schemaLinks}\nSQL:"
|
||||
new_case_prompt = new_case_template.format(question=user_query, currentDate=data_date, tableName=domain_name, schemaLinks=schema_link_str)
|
||||
|
||||
sql_example_prompt = instruction + '\n\n' + sql_example_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return sql_example_prompt
|
||||
|
||||
def generate_schema_linking_sql_prompt(self, user_query: str,
|
||||
domain_name: str,
|
||||
data_date : str,
|
||||
fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str],
|
||||
fewshot_example_list:List[Mapping[str, str]]):
|
||||
|
||||
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
|
||||
|
||||
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links,再根据schema_links为每个问题生成SQL查询语句"
|
||||
|
||||
example_keys = ["tableName", "fieldsList", "priorSchemaLinks", "currentDate", "question", "analysis", "schemaLinks", "sql"]
|
||||
example_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\nCurrent_date:{currentDate}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schemaLinks}\nSQL:{sql}"
|
||||
fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=example_template,
|
||||
example_keys=example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\nCurrent_date:{currentDate}\n问题:{question}\n分析: 让我们一步一步地思考。"
|
||||
new_case_prompt = new_case_template.format(tableName=domain_name, fieldsList=fields_list, priorSchemaLinks=prior_schema_links_str, currentDate=data_date, question=user_query)
|
||||
|
||||
prompt = instruction + '\n\n' + fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return prompt
|
||||
|
||||
async def async_query2sql(self, query_text: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
data_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("query_text: {}".format(query_text))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("data_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
query_text = query_text + ' 备注:'+prior_exts
|
||||
logger.info("query_text_prior_exts: {}".format(query_text))
|
||||
|
||||
fewshot_example_meta_list = self.get_fewshot_examples(query_text, filter_condition)
|
||||
schema_linking_prompt = self.generate_schema_linking_prompt(query_text, model_name, fields_list, prior_schema_links, fewshot_example_meta_list)
|
||||
logger.debug("schema_linking_prompt->{}".format(schema_linking_prompt))
|
||||
schema_link_output = await self.llm._call_async(schema_linking_prompt)
|
||||
|
||||
schema_link_str = schema_link_parse(schema_link_output)
|
||||
|
||||
sql_prompt = self.generate_sql_prompt(query_text, model_name, schema_link_str, data_date, fewshot_example_meta_list)
|
||||
logger.debug("sql_prompt->{}".format(sql_prompt))
|
||||
sql_output = await self.llm._call_async(sql_prompt)
|
||||
|
||||
resp = dict()
|
||||
resp['query'] = query_text
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkingOutput'] = schema_link_output
|
||||
resp['schemaLinkStr'] = schema_link_str
|
||||
|
||||
resp['sqlOutput'] = sql_output
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
async def async_query2sql_shortcut(self, query_text: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
data_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
logger.info("query_text: {}".format(query_text))
|
||||
logger.info("model_name: {}".format(model_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("data_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
query_text = query_text + ' 备注:'+prior_exts
|
||||
logger.info("query_text_prior_exts: {}".format(query_text))
|
||||
|
||||
fewshot_example_meta_list = self.get_fewshot_examples(query_text, filter_condition)
|
||||
schema_linking_sql_shortcut_prompt = self.generate_schema_linking_sql_prompt(query_text, model_name, data_date, fields_list, prior_schema_links, fewshot_example_meta_list)
|
||||
logger.debug("schema_linking_sql_shortcut_prompt->{}".format(schema_linking_sql_shortcut_prompt))
|
||||
schema_linking_sql_shortcut_output = await self.llm._call_async(schema_linking_sql_shortcut_prompt)
|
||||
|
||||
schema_linking_str = combo_schema_link_parse(schema_linking_sql_shortcut_output)
|
||||
sql_str = combo_sql_parse(schema_linking_sql_shortcut_output)
|
||||
|
||||
resp = dict()
|
||||
resp['query'] = query_text
|
||||
resp['model'] = model_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkingComboOutput'] = schema_linking_sql_shortcut_output
|
||||
resp['schemaLinkStr'] = schema_linking_str
|
||||
resp['sqlOutput'] = sql_str
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
class Text2DSLAgentConsistency(object):
|
||||
def __init__(self, num_fewshots:int, num_examples:int, num_self_consistency:int,
|
||||
sql_example_prompter:FewShotPromptTemplate2, llm: BaseLLM) -> None:
|
||||
self.num_fewshots = num_fewshots
|
||||
self.num_examples = num_examples
|
||||
assert self.num_fewshots <= self.num_examples
|
||||
self.num_self_consistency = num_self_consistency
|
||||
|
||||
self.llm = llm
|
||||
self.sql_example_prompter = sql_example_prompter
|
||||
|
||||
def reload_setting(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]], num_examples:int, num_fewshots:int, num_self_consistency:int):
|
||||
self.num_fewshots = num_fewshots
|
||||
self.num_examples = num_examples
|
||||
assert self.num_fewshots <= self.num_examples
|
||||
self.num_self_consistency = num_self_consistency
|
||||
assert self.num_self_consistency >= 1
|
||||
self.sql_example_prompter.reload_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def add_examples(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]]):
|
||||
self.sql_example_prompter.add_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def update_examples(self, sql_example_ids:List[str], sql_example_units: List[Mapping[str, str]]):
|
||||
self.sql_example_prompter.update_few_shot_example(sql_example_ids, sql_example_units)
|
||||
|
||||
def delete_examples(self, sql_example_ids:List[str]):
|
||||
self.sql_example_prompter.delete_few_shot_example(sql_example_ids)
|
||||
|
||||
def count_examples(self):
|
||||
return self.sql_example_prompter.count_few_shot_example()
|
||||
|
||||
def get_examples_candidates(self, query_text: str, filter_condition: Mapping[str, str])->List[Mapping[str, str]]:
|
||||
few_shot_example_meta_list = self.sql_example_prompter.retrieve_few_shot_example(query_text, self.num_examples, filter_condition)
|
||||
|
||||
return few_shot_example_meta_list
|
||||
|
||||
def get_fewshot_example_combos(self, example_meta_list:List[Mapping[str, str]])-> List[List[Mapping[str, str]]]:
|
||||
fewshot_example_list = []
|
||||
for i in range(0, self.num_self_consistency):
|
||||
random.shuffle(example_meta_list)
|
||||
fewshot_example_list.append(example_meta_list[:self.num_fewshots])
|
||||
|
||||
return fewshot_example_list
|
||||
|
||||
def generate_schema_linking_prompt(self, user_query: str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
|
||||
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
|
||||
|
||||
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links"
|
||||
|
||||
schema_linking_example_keys = ["tableName", "fieldsList", "priorSchemaLinks", "question", "analysis", "schemaLinks"]
|
||||
schema_linking_example_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schemaLinks}"
|
||||
schema_linking_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=schema_linking_example_template,
|
||||
example_keys=schema_linking_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "Table {tableName}, columns = {fieldsList}, prior_schema_links = {priorSchemaLinks}\n问题:{question}\n分析: 让我们一步一步地思考。"
|
||||
new_case_prompt = new_case_template.format(tableName=domain_name, fieldsList=fields_list, priorSchemaLinks=prior_schema_links_str, question=user_query)
|
||||
|
||||
schema_linking_prompt = instruction + '\n\n' + schema_linking_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
return schema_linking_prompt
|
||||
|
||||
def generate_schema_linking_prompt_pool(self, user_query: str, domain_name: str, fields_list: List[str],
|
||||
prior_schema_links: Mapping[str,str], fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
schema_linking_prompt_pool = []
|
||||
for fewshot_example_list in fewshot_example_list_pool:
|
||||
schema_linking_prompt = self.generate_schema_linking_prompt(user_query, domain_name, fields_list, prior_schema_links, fewshot_example_list)
|
||||
schema_linking_prompt_pool.append(schema_linking_prompt)
|
||||
|
||||
return schema_linking_prompt_pool
|
||||
|
||||
def generate_sql_prompt(self, user_query: str, domain_name: str,
|
||||
schema_link_str: str, data_date: str,
|
||||
fewshot_example_list:List[Mapping[str, str]])-> str:
|
||||
instruction = "# 根据schema_links为每个问题生成SQL查询语句"
|
||||
sql_example_keys = ["question", "currentDate", "tableName", "schemaLinks", "sql"]
|
||||
sql_example_template = "问题:{question}\nCurrent_date:{currentDate}\nTable {tableName}\nSchema_links:{schemaLinks}\nSQL:{sql}"
|
||||
|
||||
sql_example_fewshot_prompt = self.sql_example_prompter.make_few_shot_example_prompt(few_shot_template=sql_example_template,
|
||||
example_keys=sql_example_keys,
|
||||
few_shot_example_meta_list=fewshot_example_list)
|
||||
|
||||
new_case_template = "问题:{question}\nCurrent_date:{currentDate}\nTable {tableName}\nSchema_links:{schemaLinks}\nSQL:"
|
||||
new_case_prompt = new_case_template.format(question=user_query, currentDate=data_date, tableName=domain_name, schemaLinks=schema_link_str)
|
||||
|
||||
sql_example_prompt = instruction + '\n\n' + sql_example_fewshot_prompt + '\n\n' + new_case_prompt
|
||||
|
||||
return sql_example_prompt
|
||||
|
||||
def generate_sql_prompt_pool(self, user_query: str, domain_name: str, data_date: str,
|
||||
schema_link_str_pool: List[str], fewshot_example_list_pool:List[List[Mapping[str, str]]])-> List[str]:
|
||||
sql_prompt_pool = []
|
||||
for schema_link_str, fewshot_example_list in zip(schema_link_str_pool, fewshot_example_list_pool):
|
||||
sql_prompt = self.generate_sql_prompt(user_query, domain_name, schema_link_str, data_date, fewshot_example_list)
|
||||
sql_prompt_pool.append(sql_prompt)
|
||||
|
||||
return sql_prompt_pool
|
||||
|
||||
def self_consistency_vote(self, output_res_pool:List[str]):
|
||||
output_res_counts = Counter(output_res_pool)
|
||||
output_res_max = output_res_counts.most_common(1)[0][0]
|
||||
total_output_num = len(output_res_pool)
|
||||
|
||||
vote_percentage = {k: (v/total_output_num) for k,v in output_res_counts.items()}
|
||||
|
||||
return output_res_max, vote_percentage
|
||||
|
||||
def schema_linking_list_str_unify(self, schema_linking_list: List[str])-> List[str]:
|
||||
schema_linking_list_unify = []
|
||||
for schema_linking_str in schema_linking_list:
|
||||
schema_linking_str_unify = ','.join(sorted([item.strip() for item in schema_linking_str.strip('[]').split(',')]))
|
||||
schema_linking_str_unify = f'[{schema_linking_str_unify}]'
|
||||
schema_linking_list_unify.append(schema_linking_str_unify)
|
||||
|
||||
return schema_linking_list_unify
|
||||
|
||||
|
||||
async def generate_schema_linking_tasks(self, user_query: str, domain_name: str,
|
||||
fields_list: List[str], prior_schema_links: Mapping[str,str],
|
||||
fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
|
||||
schema_linking_prompt_pool = self.generate_schema_linking_prompt_pool(user_query, domain_name,
|
||||
fields_list, prior_schema_links,
|
||||
fewshot_example_list_combo)
|
||||
schema_linking_output_task_pool = [self.llm._call_async(schema_linking_prompt) for schema_linking_prompt in schema_linking_prompt_pool]
|
||||
schema_linking_output_res_pool = await asyncio.gather(*schema_linking_output_task_pool)
|
||||
logger.debug(f'schema_linking_output_res_pool:{schema_linking_output_res_pool}')
|
||||
|
||||
return schema_linking_output_res_pool
|
||||
|
||||
async def generate_sql_tasks(self, user_query: str, domain_name: str, data_date: str,
|
||||
schema_link_str_pool: List[str], fewshot_example_list_combo:List[List[Mapping[str, str]]]):
|
||||
|
||||
sql_prompt_pool = self.generate_sql_prompt_pool(user_query, domain_name, schema_link_str_pool, data_date, fewshot_example_list_combo)
|
||||
sql_output_task_pool = [self.llm._call_async(sql_prompt) for sql_prompt in sql_prompt_pool]
|
||||
sql_output_res_pool = await asyncio.gather(*sql_output_task_pool)
|
||||
logger.debug(f'sql_output_res_pool:{sql_output_res_pool}')
|
||||
|
||||
return sql_output_res_pool
|
||||
|
||||
async def tasks_run(self, user_query: str, filter_condition: Mapping[str, str], domain_name: str, fields_list: List[str], prior_schema_links: Mapping[str,str], data_date: str, prior_exts: str):
|
||||
logger.info("user_query: {}".format(user_query))
|
||||
logger.info("domain_name: {}".format(domain_name))
|
||||
logger.info("fields_list: {}".format(fields_list))
|
||||
logger.info("current_date: {}".format(data_date))
|
||||
logger.info("prior_schema_links: {}".format(prior_schema_links))
|
||||
logger.info("prior_exts: {}".format(prior_exts))
|
||||
|
||||
if prior_exts != '':
|
||||
user_query = user_query + ' 备注:'+prior_exts
|
||||
logger.info("user_query_prior_exts: {}".format(user_query))
|
||||
|
||||
fewshot_example_meta_list = self.get_examples_candidates(user_query, filter_condition)
|
||||
fewshot_example_list_combo = self.get_fewshot_example_combos(fewshot_example_meta_list)
|
||||
|
||||
schema_linking_output_candidates = await self.generate_schema_linking_tasks(user_query, domain_name, fields_list, prior_schema_links, fewshot_example_list_combo)
|
||||
schema_linking_candidate_list = [schema_link_parse(schema_linking_output_candidate) for schema_linking_output_candidate in schema_linking_output_candidates]
|
||||
logger.debug(f'schema_linking_candidate_list:{schema_linking_candidate_list}')
|
||||
schema_linking_candidate_sorted_list = self.schema_linking_list_str_unify(schema_linking_candidate_list)
|
||||
logger.debug(f'schema_linking_candidate_sorted_list:{schema_linking_candidate_sorted_list}')
|
||||
|
||||
schema_linking_output_max, schema_linking_output_vote_percentage = self.self_consistency_vote(schema_linking_candidate_sorted_list)
|
||||
|
||||
sql_output_candicates = await self.generate_sql_tasks(user_query, domain_name, data_date, schema_linking_candidate_list,fewshot_example_list_combo)
|
||||
logger.debug(f'sql_output_candicates:{sql_output_candicates}')
|
||||
sql_output_max, sql_output_vote_percentage = self.self_consistency_vote(sql_output_candicates)
|
||||
|
||||
resp = dict()
|
||||
resp['query'] = user_query
|
||||
resp['model'] = domain_name
|
||||
resp['fields'] = fields_list
|
||||
resp['priorSchemaLinking'] = prior_schema_links
|
||||
resp['dataDate'] = data_date
|
||||
|
||||
resp['schemaLinkStr'] = schema_linking_output_max
|
||||
resp['schemaLinkingWeight'] = schema_linking_output_vote_percentage
|
||||
|
||||
resp['sqlOutput'] = sql_output_max
|
||||
resp['sqlWeight'] = sql_output_vote_percentage
|
||||
|
||||
logger.info("resp: {}".format(resp))
|
||||
|
||||
return resp
|
||||
|
||||
class Text2DSLAgentWrapper(object):
|
||||
def __init__(self, sql_agent:Text2DSLAgent, sql_agent_cs:Text2DSLAgentConsistency,
|
||||
is_shortcut:bool, is_self_consistency:bool):
|
||||
self.sql_agent = sql_agent
|
||||
self.sql_agent_cs = sql_agent_cs
|
||||
|
||||
self.is_shortcut = is_shortcut
|
||||
self.is_self_consistency = is_self_consistency
|
||||
|
||||
async def async_query2sql(self, query_text: str, filter_condition: Mapping[str,str],
|
||||
model_name: str, fields_list: List[str],
|
||||
data_date: str, prior_schema_links: Mapping[str,str], prior_exts: str):
|
||||
if self.is_self_consistency:
|
||||
logger.info("sql wrapper: self_consistency")
|
||||
resp = await self.sql_agent_cs.tasks_run(user_query=query_text, filter_condition=filter_condition, domain_name=model_name, fields_list=fields_list, prior_schema_links=prior_schema_links, data_date=data_date, prior_exts=prior_exts)
|
||||
return resp
|
||||
elif self.is_shortcut:
|
||||
logger.info("sql wrapper: shortcut")
|
||||
resp = await self.sql_agent.async_query2sql_shortcut(query_text=query_text, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list, data_date=data_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
return resp
|
||||
else:
|
||||
logger.info("sql wrapper: normal")
|
||||
resp = await self.sql_agent.async_query2sql(query_text=query_text, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list, data_date=data_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
return resp
|
||||
|
||||
def update_configs(self, is_shortcut, is_self_consistency,
|
||||
sql_examplars, num_examples, num_fewshots, num_self_consistency):
|
||||
self.is_shortcut = is_shortcut
|
||||
self.is_self_consistency = is_self_consistency
|
||||
|
||||
self.sql_agent.update_examples(sql_examplars=sql_examplars, num_fewshots=num_examples)
|
||||
self.sql_agent_cs.update_examples(sql_examplars=sql_examplars, num_examples=num_examples, num_fewshots=num_fewshots, num_self_consistency=num_self_consistency)
|
||||
|
||||
@@ -8,7 +8,7 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from services.sql.run import text2sql_agent_router
|
||||
from services.s2ql.run import text2sql_agent_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -45,12 +45,17 @@ async def query2sql(query_body: Mapping[str, Any]):
|
||||
else:
|
||||
filter_condition = query_body['filterCondition']
|
||||
|
||||
if 'sql_generation_mode' not in query_body:
|
||||
raise HTTPException(status_code=400, detail="sql_generation_mode is not in query_body")
|
||||
else:
|
||||
sql_generation_mode = query_body['sql_generation_mode']
|
||||
|
||||
model_name = schema['modelName']
|
||||
fields_list = schema['fieldNameList']
|
||||
prior_schema_links = {item['fieldValue']:item['fieldName'] for item in linking}
|
||||
|
||||
resp = await text2sql_agent_router.async_query2sql(query_text=query_text, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list,
|
||||
data_date=current_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts)
|
||||
resp = await text2sql_agent_router.async_query2sql(question=query_text, filter_condition=filter_condition, model_name=model_name, fields_list=fields_list,
|
||||
data_date=current_date, prior_schema_links=prior_schema_links, prior_exts=prior_exts, sql_generation_mode=sql_generation_mode)
|
||||
|
||||
return resp
|
||||
|
||||
@@ -82,18 +87,7 @@ def query2sql_setting_update(query_body: Mapping[str, Any]):
|
||||
else:
|
||||
self_consistency_nums = query_body['selfConsistencyNums']
|
||||
|
||||
if 'isShortcut' not in query_body:
|
||||
raise HTTPException(status_code=400, detail="isShortcut is not in query_body")
|
||||
else:
|
||||
is_shortcut = query_body['isShortcut']
|
||||
|
||||
if 'isSelfConsistency' not in query_body:
|
||||
raise HTTPException(status_code=400, detail="isSelfConsistency is not in query_body")
|
||||
else:
|
||||
is_self_consistency = query_body['isSelfConsistency']
|
||||
|
||||
text2sql_agent_router.update_configs(is_shortcut=is_shortcut, is_self_consistency=is_self_consistency,
|
||||
sql_example_ids=sql_ids, sql_example_units=sql_examplars,
|
||||
text2sql_agent_router.update_configs(sql_example_ids=sql_ids, sql_example_units=sql_examplars,
|
||||
num_examples=example_nums, num_fewshots=fewshot_nums, num_self_consistency=self_consistency_nums)
|
||||
|
||||
return "success"
|
||||
@@ -112,8 +106,7 @@ def query2sql_add_examples(query_body: Mapping[str, Any]):
|
||||
else:
|
||||
sql_examplars = query_body['sqlExamplars']
|
||||
|
||||
text2sql_agent_router.sql_agent.add_examples(sql_example_ids=sql_ids, sql_example_units=sql_examplars)
|
||||
text2sql_agent_router.sql_agent_cs.add_examples(sql_example_ids=sql_ids, sql_example_units=sql_examplars)
|
||||
text2sql_agent_router.add_examples(sql_example_ids=sql_ids, sql_example_units=sql_examplars)
|
||||
|
||||
return "success"
|
||||
|
||||
@@ -131,8 +124,7 @@ def query2sql_update_examples(query_body: Mapping[str, Any]):
|
||||
else:
|
||||
sql_examplars = query_body['sqlExamplars']
|
||||
|
||||
text2sql_agent_router.sql_agent.update_examples(sql_example_ids=sql_ids, sql_example_units=sql_examplars)
|
||||
text2sql_agent_router.sql_agent_cs.update_examples(sql_example_ids=sql_ids, sql_example_units=sql_examplars)
|
||||
text2sql_agent_router.update_examples(sql_example_ids=sql_ids, sql_example_units=sql_examplars)
|
||||
|
||||
return "success"
|
||||
|
||||
@@ -144,18 +136,24 @@ def query2sql_delete_examples(query_body: Mapping[str, Any]):
|
||||
else:
|
||||
sql_ids = query_body['sqlIds']
|
||||
|
||||
text2sql_agent_router.sql_agent.delete_examples(sql_example_ids=sql_ids)
|
||||
text2sql_agent_router.sql_agent_cs.delete_examples(sql_example_ids=sql_ids)
|
||||
text2sql_agent_router.delete_examples(sql_example_ids=sql_ids)
|
||||
|
||||
return "success"
|
||||
|
||||
@router.post("/query2sql_get_examples")
|
||||
def query2sql_get_examples(query_body: Mapping[str, Any]):
|
||||
if 'sqlIds' not in query_body:
|
||||
raise HTTPException(status_code=400, detail="sqlIds is not in query_body")
|
||||
else:
|
||||
sql_ids = query_body['sqlIds']
|
||||
|
||||
examples = text2sql_agent_router.get_examples(sql_example_ids=sql_ids)
|
||||
|
||||
return examples
|
||||
|
||||
@router.get("/query2sql_count_examples")
|
||||
def query2sql_count_examples():
|
||||
sql_agent_examples_cnt = text2sql_agent_router.sql_agent.count_examples()
|
||||
sql_agent_cs_examples_cnt = text2sql_agent_router.sql_agent_cs.count_examples()
|
||||
examples_cnt = text2sql_agent_router.count_examples()
|
||||
|
||||
assert sql_agent_examples_cnt == sql_agent_cs_examples_cnt
|
||||
|
||||
return sql_agent_examples_cnt
|
||||
return examples_cnt
|
||||
|
||||
|
||||
Reference in New Issue
Block a user