From 45fb83356fba075bebb74a65326df45122879b74 Mon Sep 17 00:00:00 2001 From: tristanliu <37809633+sevenliu1896@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:24:00 +0800 Subject: [PATCH] [improvement][headless-fe] Unified the SQL generation method for data sets to ensure consistency and improve efficiency. (#1520) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [improvement][semantic-fe] add metric trend download functionality * [improvement][semantic-fe] fix the dimension initialization issue in metric correlation * [improvement][semantic-fe] Fix the issue of database changes not taking effect when creating based on an SQL data source. * [improvement][semantic-fe] Optimizing pagination logic and some CSS styles * [improvement][semantic-fe] Fixing the API for the indicator list by changing "current" to "pageNum" * [improvement][semantic-fe] Fixing the default value setting for the indicator list * [improvement][semantic-fe] Adding batch operations for indicators/dimensions/models * [improvement][semantic-fe] Replacing the single status update API for indicators/dimensions with a batch update API * [improvement][semantic-fe] Redesigning the indicator homepage to incorporate trend charts and table functionality for indicators * [improvement][semantic-fe] Optimizing the logic for setting dimension values and editing data sources, and adding system settings functionality * [improvement][semantic-fe] Upgrading antd version to 5.x, extracting the batch operation button component, optimizing the interaction for system settings, and expanding the configuration generation types for list-to-select component. * [improvement][semantic-fe] Adding the ability to filter dimensions based on whether they are tags or not. * [improvement][semantic-fe] Adding the ability to edit relationships between models in the canvas. * [improvement][semantic-fe] Updating the datePicker component to use dayjs instead. * [improvement][semantic-fe] Fixing the issue with passing the model ID for dimensions in the indicator market. * [improvement][semantic-fe] Fixing the abnormal state of the popup when creating a model. * [improvement][semantic-fe] Adding permission logic for bulk operations in the indicator market. * [improvement][semantic-fe] Adding the ability to download and transpose data. * [improvement][semantic-fe] Fixing the initialization issue with the date selection component in the indicator details page when switching time granularity. * [improvement][semantic-fe] Fixing the logic error in the dimension value setting. * [improvement][semantic-fe] Fixing the synchronization issue with the question and answer settings information. * [improvement][semantic-fe] Optimizing the canvas functionality for better performance and user experience. * [improvement][semantic-fe] Optimizing the update process for drawing model relationship edges in the canvas. * [improvement][semantic-fe] Changing the line type for canvas connections. * [improvement][semantic-fe] Replacing the initialization variable from "semantic" to "headless". * [improvement][semantic-fe] Fixing the missing migration issue for default drill-down dimension configuration in model editing. Additionally, optimizing the data retrieval method for initializing fields in the model. * [improvement][semantic-fe] Updating the logic for the fieldName. * [improvement][semantic-fe] Adjusting the position of the metrics tab. * [improvement][semantic-fe] Changing the 字段名称 to 英文名称. * [improvement][semantic-fe] Fix metric measurement deletion. * [improvement][semantic-fe] UI optimization for metric details page. * [improvement][semantic-fe] UI optimization for metric details page. * [improvement][semantic-fe] UI adjustment for metric details page. * [improvement][semantic-fe] The granularity field in the time type of model editing now supports setting it as empty. * [improvement][semantic-fe] Added field type and metric type to the metric creation options. * [improvement][semantic-fe] The organization structure selection feature has been added to the permission management. * [improvement][semantic-fe] Improved user experience for the metric list. * [improvement][semantic-fe] fix update the metric list. * [improvement][headless-fe] Added view management functionality. * [improvement][headless-fe] The view management functionality has been added. This feature allows users to create, edit, and manage different views within the system. * [improvement][headless-fe] Added model editing side effect detection. * [improvement][headless-fe] Fixed the logic error in view editing. * [improvement][headless-fe] Fixed the issue with initializing dimension associations in metric settings. * [improvement][headless-fe] Added the ability to hide the Q&A settings entry point. * [improvement][headless-fe] Fixed the issue with selecting search results in metric field creation. * [improvement][headless-fe] Added search functionality to the field list in model editing. * [improvement][headless-fe] fix the field list in model editing * [improvement][headless-fe] Restructured the data for the dimension value settings interface. * [improvement][headless-fe] Added dynamic variable functionality to model creation based on SQL scripts. * [improvement][headless-fe] Added support for passing dynamic variables as parameters in the executeSql function. * [improvement][headless-fe] Resolved the issue where users were unable to select all options for dimensions, metrics, and fields in the metric generation process. * [improvement][headless-fe] Replaced the term "view" with "dataset" * [improvement][headless-fe] Added the ability to export metrics and dimensions to a specific target. * [improvement][headless-fe] Enhanced dataset creation to support the tag mode. * [improvement][headless-fe] Added tag value setting. * [improvement][headless-fe] Optimized the tag setting system. * [improvement][headless-fe] Optimized the tag setting system. * [improvement][headless-fe] Updated the data initialization for model editing to use API requests instead. * [improvement][headless-fe] Added search functionality to model management. * [improvement][headless-fe] Removed field null validation during model editing. * [improvement][headless-fe] Updated the batch operation button component. * [improvement][headless-fe] Optimized the logic for initializing indicators in dimension value settings. * [improvement][headless-fe] Adjusted the length of the input field for model editing names. * [improvement][headless-fe] Lock the version of the @ant-design/pro-table component and replace it with @ant-design/pro-components. * [improvement][headless-fe] Optimized the style of the metrics market and tags market. * [improvement][headless-fe] The quick creation of model fields now defaults to using the "comment" field for filling. * [improvement][headless-fe] The quick creation of model fields now defaults to using the "comment" field for filling * [improvement][headless-fe] The quick creation of model fields now defaults to using the "comment" field for filling. * [improvement][headless-fe] Fixed the issue where the conditions for metric measurement creation were not being saved correctly. * [improvement][headless-fe] Default value setting for hiding dimensions. * [improvement][headless-fe] Updated the file imports in the project. * [improvement][headless-fe] Adjusted the logic for displaying the tab in the theme domain. * [improvement][headless-fe] Added term management functionality. * [improvement][headless-fe] When creating a model, the current metric operator now allows for clearance. * [improvement][headless-fe] Term management interface transformation * [improvement][headless-fe] Migrating scaffold version to @umi/max * [improvement][headless-fe] remove modle column * [improvement][headless-fe] 1.Added configuration for the large language model in the agent; 2.upgraded React version from 17 to 18; 3.modified some UI effects. * [improvement][headless-fe] Added a simplified mode to the question-answering system. * [improvement][headless-fe] remove pnpm-lock * [improvement][headless-fe] add pnpm-lock * [improvement][headless-fe] Fixed the issue with passing the modelId during initialization. * [improvement][headless-fe] Fixed the issue with abnormal comments during model creation. * [improvement][headless-fe] fix build bugs * [improvement][headless-fe] change build config * [improvement][headless-fe] route config change * [improvement][headless-fe] Optimized data updating when switching domains. * [improvement][headless-fe] css change * [improvement][semantic-fe] logo css change * [improvement][semantic-fe] system config add defaultValue support * [improvement][semantic-fe] tag mode wording change * [improvement][semantic-fe] fix metric edit page init error * [improvement][supersonic-fe] Updated the way chat projects are imported in supersonic-fe. * [improvement][chat-engine] Added a background silent mode for watching chat projects. * [improvement][supersonic-fe] fix proxy url * [improvement][headless-fe] agent setting update * [improvement][headless-fe] Agent configuration: Added connectivity testing for large models. * [improvement][headless-fe] Chat: Enabled integration with agent configuration. * [improvement][headless-fe] SQL formatter: Updated the import method. * [improvement][headless-fe] login fixed * [improvement][headless-fe] Agent: Optimized the logic for saving. * [improvement][headless-fe] Model management: Integrated into the main theme domain. * [improvement][headless-fe] Sensitivity: Added bulk modification functionality. * [improvement][headless-fe] wording change * [improvement][headless-fe] Prompt configuration: Added the ability to configure prompts. * [improvement][headless-fe] Added the ability to configure embedding models. * [improvement][headless-fe] hidden configure embedding models. * [improvement][headless-fe] Connection test prompt update for large language model settings. * [improvement][headless-fe] add memory review config * [improvement][headless-fe] Rollback of data structure for large language model configuration. * [improvement][headless-fe] Added dependency relationships between various items in system configuration. * [improvement][headless-fe] Added password parsing to the generation rules of system settings. * [improvement][headless-fe] Added slider parsing to the generation rules of system settings. * [improvement][headless-fe] Optimized the logic for initializing values in system settings. * [improvement][headless-fe] Optimized the time format * [improvement][headless-fe] Unified the SQL generation method for data sets to ensure consistency and improve efficiency. --------- Co-authored-by: tristanliu --- .../Datasource/components/ModelBasicForm.tsx | 15 +++---- .../Datasource/components/ModelCreateForm.tsx | 7 +-- .../Datasource/components/ModelFieldForm.tsx | 1 + .../Datasource/components/SqlDetail.tsx | 43 ++++++++++++------- .../pages/SemanticModel/Datasource/data.d.ts | 4 +- .../SemanticModel/models/dimensionData.ts | 2 +- .../pages/SemanticModel/models/metricData.ts | 2 +- .../src/pages/SemanticModel/service.ts | 30 ++++++++++--- 8 files changed, 68 insertions(+), 36 deletions(-) diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx index c20788537..6d12e115c 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx @@ -4,7 +4,6 @@ import type { FormInstance } from 'antd/lib/form'; import { getDbNames, getTables, getDimensionList } from '../../service'; import { ISemantic } from '../../data'; import FormItemTitle from '@/components/FormHelper/FormItemTitle'; -import { isString } from 'lodash'; const FormItem = Form.Item; const { TextArea } = Input; @@ -24,7 +23,7 @@ const ModelBasicForm: React.FC = ({ mode = 'normal', }) => { const [currentDbLinkConfigId, setCurrentDbLinkConfigId] = useState(); - const [dbNameList, setDbNameList] = useState([]); + const [dbNameList, setDbNameList] = useState([]); const [tableNameList, setTableNameList] = useState([]); const [loading, setLoading] = useState(false); const [dimensionOptions, setDimensionOptions] = useState<{ label: string; value: number }[]>([]); @@ -56,7 +55,7 @@ const ModelBasicForm: React.FC = ({ const { code, data, msg } = await getDbNames(databaseId); setLoading(false); if (code === 200) { - const list = data?.resultList || []; + const list = data || []; setDbNameList(list); } else { message.error(msg); @@ -70,7 +69,7 @@ const ModelBasicForm: React.FC = ({ const { code, data, msg } = await getTables(currentDbLinkConfigId, databaseName); setLoading(false); if (code === 200) { - const list = data?.resultList || []; + const list = data || []; setTableNameList(list); } else { message.error(msg); @@ -116,8 +115,8 @@ const ModelBasicForm: React.FC = ({ }} > {dbNameList.map((item) => ( - - {item.name} + + {item} ))} @@ -129,8 +128,8 @@ const ModelBasicForm: React.FC = ({ > diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx index 5922b574b..01bd94572 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx @@ -285,6 +285,7 @@ const ModelCreateForm: React.FC = ({ setFields(fieldsClassifyList || []); return; } + const columnFields: any[] = columns.map((item: IDataSource.IExecuteSqlColumn) => { const { type, nameEn, comment } = item; const oldItem = @@ -404,12 +405,12 @@ const ModelCreateForm: React.FC = ({ const queryTableColumnList = async (databaseId: number, dbName: string, tableName: string) => { const { code, data, msg } = await getColumns(databaseId, dbName, tableName); if (code === 200) { - const list = data?.resultList || []; + const list = data || []; const columns = list.map((item: any, index: number) => { - const { dataType, name, comment } = item; + const { dataType, columnName, comment } = item; return { comment, - nameEn: name, + nameEn: columnName, type: dataType, }; }); diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx index 949fee58f..f2e281d1a 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx @@ -327,6 +327,7 @@ const ModelFieldForm: React.FC = ({ return item.bizName.includes(filterValue); }) || [] : fields; + return ( <>
diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlDetail.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlDetail.tsx index 5ad5968cb..963103f92 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlDetail.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlDetail.tsx @@ -18,8 +18,8 @@ import { isFunction } from 'lodash'; import FullScreen from '@/components/FullScreen'; import SqlEditor from '@/components/SqlEditor'; import type { TaskResultItem, TaskResultColumn } from '../data'; -import { excuteSql } from '@/pages/SemanticModel/service'; -import type { StateType } from '../../model'; +import { executeSql, getColumnsBySql } from '@/pages/SemanticModel/service'; + import SqlParams from './SqlParams'; import styles from '../style.less'; import 'ace-builds/src-min-noconflict/ext-searchbox'; @@ -104,7 +104,7 @@ const SqlDetail: React.FC = ({ const [variableCollapsed, setVariableCollapsed] = useState(true); const [sqlParams, setSqlParams] = useState([]); - const [scriptColumns, setScriptColumns] = useState([]); + const [scriptColumns, setScriptColumns] = useState([]); useEffect(() => { const list = databaseConfigList.map((item: ISemantic.IDatabaseItem) => { @@ -186,21 +186,21 @@ const SqlDetail: React.FC = ({ const updateResultCols = (list: TaskResultItem[], columns: TaskResultColumn[]) => { if (list.length) { const widthMap = getKeyWidthMap(list); - const cols = columns.map(({ nameEn }) => { + const cols = columns.map(({ columnName }) => { return { - key: nameEn, - title: nameEn, - dataIndex: nameEn, - width: `${(widthMap[nameEn] as number) + 22}px`, // 字宽度 + 20px(比左右padding宽几像素，作为一个buffer值) + key: columnName, + title: columnName, + dataIndex: columnName, + width: `${(widthMap[columnName] as number) + 22}px`, // 字宽度 + 20px(比左右padding宽几像素，作为一个buffer值) }; }); setResultCols(cols); } }; - const fetchTaskResult = (params) => { + const fetchTaskResult = (params: any, columnData = []) => { setResultTable( - params.resultList.map((item, index) => { + params.resultList.map((item: Record, index: number) => { return { ...item, index, @@ -212,8 +212,8 @@ const SqlDetail: React.FC = ({ pageSize: 20, total: params.resultList.length, }); - setScriptColumns(params.columns); - updateResultCols(params.resultList, params.columns); + setScriptColumns(columnData); + updateResultCols(params.resultList, columnData); }; const changePaging = (paging: Pagination) => { @@ -243,14 +243,27 @@ const SqlDetail: React.FC = ({ return; } setResultTableLoading(true); - const { code, data, msg } = await excuteSql({ + const { code, data, msg } = await executeSql({ sql: value, id: currentDatabaseItem.key, sqlVariables: sqlParams, }); + + const { code: getColumnCode, data: getColumnData } = await getColumnsBySql({ + sql: value, + databaseId: currentDatabaseItem.key, + }); + setResultTableLoading(false); - if (code === 200) { - fetchTaskResult(data); + if (code === 200 && getColumnCode === 200) { + const columnData = getColumnData.map((item) => { + return { + ...item, + nameEn: item.columnName, + type: item.dataType, + }; + }); + fetchTaskResult(data, columnData); setRunState(true); } else { setRunState(false); diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/data.d.ts b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/data.d.ts index 14fe0869d..9bd8400b6 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/data.d.ts +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/data.d.ts @@ -1,7 +1,7 @@ // 任务查询结果列 export type TaskResultColumn = { - name: string; - type: string; + columnName: string; + dataType: string; }; // 任务查询结果 diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/dimensionData.ts b/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/dimensionData.ts index b1241deb0..37a44c049 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/dimensionData.ts +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/dimensionData.ts @@ -1,7 +1,7 @@ import { message } from 'antd'; import { ISemantic } from '../data'; import { useState } from 'react'; -import { getDimensionList, queryMetric, excuteSql, getDatabaseList } from '../service'; +import { getDimensionList } from '../service'; export default function Dimension() { const [dimensionList, setDimensionList] = useState([]); diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/metricData.ts b/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/metricData.ts index eeb7e3213..caab8b12e 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/metricData.ts +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/models/metricData.ts @@ -1,7 +1,7 @@ import { message } from 'antd'; import { ISemantic } from '../data'; import { useState } from 'react'; -import { queryMetric, excuteSql, getDatabaseList } from '../service'; +import { queryMetric } from '../service'; export default function Metric() { const [metricList, setMetricList] = useState([]); diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts b/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts index 3bd09f354..0352a0b95 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts @@ -359,26 +359,44 @@ type ExcuteSqlParams = { }; // 执行脚本 -export async function excuteSql(params: ExcuteSqlParams) { +export async function executeSql(params: ExcuteSqlParams) { const data = { ...params }; return request.post(`${process.env.API_BASE_URL}database/executeSql`, { data }); } +export async function getColumnsBySql(params: { databaseId: number; sql: string }) { + return request.get(`${process.env.API_BASE_URL}database/getColumnsBySql`, { + params, + }); +} + export function getDbNames(dbId: number): Promise { - return request(`${process.env.API_BASE_URL}database/getDbNames/${dbId}`, { + return request(`${process.env.API_BASE_URL}database/getDbNames`, { method: 'GET', + params: { + id: dbId, + }, }); } -export function getTables(dbId: number, dbName: string): Promise { - return request(`${process.env.API_BASE_URL}database/getTables/${dbId}/${dbName}`, { +export function getTables(databaseId: number, dbName: string): Promise { + return request(`${process.env.API_BASE_URL}database/getTables`, { method: 'GET', + params: { + databaseId, + db: dbName, + }, }); } -export function getColumns(dbId: number, dbName: string, tableName: string): Promise { - return request(`${process.env.API_BASE_URL}database/getColumns/${dbId}/${dbName}/${tableName}`, { +export function getColumns(databaseId: number, dbName: string, tableName: string): Promise { + return request(`${process.env.API_BASE_URL}database/getColumnsByName`, { method: 'GET', + params: { + databaseId, + db: dbName, + table: tableName, + }, }); }