when sql script has variables, generate sql can't be executed, this commit supports variable replement (#2259)

* (fix)(headless)修复生成的SQL字段不在数据集内，导致查询异常的问题，由原来的字段数量判断更改为查询字段是否包含维度和度量字段

* (fix)(headless) <修复开启多轮对话会使用默认的gpt-4o-mini账号去进行过程检测而不是使用配置好的本地大模型的问题>

* 修复维度值导入字典时，维度值和度量值为英文名导致字段不匹配的问题

修复维度值导入字典时，维度值和度量值为英文名导致字段不匹配的问题。
维度值导入字典时会遍历调用到SqlQueryParser方法，这时的维度值或度量值为Bizname而非name，新增一个set用于添加Bizname，并在判断中新加入Bizname名的判断，只要两个条件符合一个即可

---------

Co-authored-by: unknown <qiji2@DC=corp,DC=conbagroup,DC=com.com>
Co-authored-by: QJ_wonder <38885395+xinpiannanhai@users.noreply.github.com>

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java

@@ -2,11 +2,17 @@ package com.tencent.supersonic.headless.api.pojo;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.headless.api.pojo.enums.VariableValueType;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 import java.util.List;
 
 @Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
 public class SqlVariable {
     private String name;
     private VariableValueType valueType;

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java

@@ -167,7 +167,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
             variable.put("retrievedInfo", JSONObject.toJSONString(results));
 
             Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable);
-            ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel();
+            ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatQueryContext.getRequest().getChatAppConfig().get("REWRITE_MULTI_TURN").getChatModelConfig());
             String response = chatLanguageModel.generate(prompt.toUserMessage().singleText());
 
             if (StringUtils.isBlank(response)) {

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java

@@ -47,12 +47,17 @@ public class SqlQueryParser implements QueryParser {
         SqlQuery sqlQuery = queryStatement.getSqlQuery();
         List<String> queryFields = SqlSelectHelper.getAllSelectFields(sqlQuery.getSql());
         Set<String> queryAliases = SqlSelectHelper.getAliasFields(sqlQuery.getSql());
+        Set<String> ontologyMetricsDimensions = Collections.synchronizedSet(new HashSet<String>());
+        Set<String> ontologyBizNameMetricsDimensions = Collections.synchronizedSet(new HashSet<String>());
         queryFields.removeAll(queryAliases);
         Ontology ontology = queryStatement.getOntology();
         OntologyQuery ontologyQuery = buildOntologyQuery(ontology, queryFields);
+        Set<String> queryFieldsSet = new HashSet<>(queryFields);
+        ontologyQuery.getMetrics().forEach(m -> {ontologyMetricsDimensions.add(m.getName()); ontologyBizNameMetricsDimensions.add(m.getBizName());});
+        ontologyQuery.getDimensions().forEach(d -> {ontologyMetricsDimensions.add(d.getName()); ontologyBizNameMetricsDimensions.add(d.getBizName());});
         // check if there are fields not matched with any metric or dimension
-        if (queryFields.size() > ontologyQuery.getMetrics().size()
-                + ontologyQuery.getDimensions().size()) {
+
+        if (!(queryFieldsSet.containsAll(ontologyMetricsDimensions)||queryFieldsSet.containsAll(ontologyBizNameMetricsDimensions))) {
             List<String> semanticFields = Lists.newArrayList();
             ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
             ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java

@@ -14,6 +14,7 @@ import com.tencent.supersonic.headless.core.pojo.JoinRelation;
 import com.tencent.supersonic.headless.core.pojo.Ontology;
 import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
 import com.tencent.supersonic.headless.core.translator.parser.Constants;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlNode;
@@ -34,6 +35,10 @@ public class DataModelNode extends SemanticNode {
         if (dataModel.getModelDetail().getSqlQuery() != null
                 && !dataModel.getModelDetail().getSqlQuery().isEmpty()) {
             sqlTable = dataModel.getModelDetail().getSqlQuery();
+            // if model has sqlVariables, parse sqlVariables
+            if (Objects.nonNull(dataModel.getModelDetail().getSqlVariables()) || (CollectionUtils.isEmpty(dataModel.getModelDetail().getSqlVariables()))) {
+                sqlTable = SqlVariableParseUtils.parse(sqlTable, dataModel.getModelDetail().getSqlVariables(), Lists.newArrayList());
+            }
         } else if (dataModel.getModelDetail().getTableQuery() != null
                 && !dataModel.getModelDetail().getTableQuery().isEmpty()) {
             if (dataModel.getModelDetail().getDbType()

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -39,6 +39,7 @@ public class ModelYamlManager {
         dataModelYamlTpl.setFilterSql(modelDetail.getFilterSql());
         dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
         dataModelYamlTpl.setId(modelResp.getId());
+        dataModelYamlTpl.setSqlVariables(modelDetail.getSqlVariables());
         return dataModelYamlTpl;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -102,7 +102,7 @@ public class SemanticSchemaManager {
         modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
         modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));
         modelDetail.getFields().addAll(d.getFields());
-
+        modelDetail.getSqlVariables().addAll(d.getSqlVariables());
         return dataModel;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/StarrocksParametersBuilder.java

@@ -15,7 +15,7 @@ public class StarrocksParametersBuilder extends DefaultParametersBuilder {
         List<DatabaseParameter> databaseParameters = new ArrayList<>();
         DatabaseParameter host = new DatabaseParameter();
         host.setComment("JDBC连接");
-        host.setValue("jdbc:mysql://localhost:3306/dbname");
+        host.setValue("jdbc:mysql://localhost:9030/dbname");
         host.setName("url");
         host.setPlaceholder("请输入JDBC连接串");
         databaseParameters.add(host);

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.server.pojo.yaml;
 
 import com.tencent.supersonic.headless.api.pojo.Field;
+import com.tencent.supersonic.headless.api.pojo.SqlVariable;
 import com.tencent.supersonic.headless.api.pojo.enums.ModelSourceType;
 import lombok.Data;
 
@@ -32,4 +33,6 @@ public class DataModelYamlTpl {
     private List<Field> fields;
 
     private ModelSourceType modelSourceTypeEnum;
+
+    private List<SqlVariable> sqlVariables;
 }

webapp/packages/chat-sdk/src/components/ChatMsg/index.tsx

@@ -122,7 +122,7 @@ const ChatMsg: React.FC<Props> = ({
     }
 
     const isMetricPie =
-      metricFields.length > 0 &&
+      categoryField.length > 0 &&
       metricFields?.length === 1 &&
       (isMobile ? dataSource?.length <= 5 : dataSource?.length <= 10) &&
       dataSource.every(item => item[metricFields[0].bizName] > 0);

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx

@@ -299,21 +299,22 @@ const ModelCreateForm: React.FC<CreateFormProps> = ({
       return;
     }
 
-    const columnFields: any[] = columns.map((item: IDataSource.IExecuteSqlColumn) => {
+    const fieldMap = new Map(
+      fieldsClassifyList.map(field => [field.fieldName, field])
+    );
+    columns.forEach((item: IDataSource.IExecuteSqlColumn) => {
       const { type, nameEn, comment } = item;
-      const oldItem =
-        fieldsClassifyList.find((oItem) => {
-          return oItem.fieldName === item.nameEn;
-        }) || {};
-      return {
-        ...oldItem,
+      const existingField = fieldMap.get(nameEn);
+
+      fieldMap.set(nameEn, {
+        ...existingField,
         bizName: nameEn,
         fieldName: nameEn,
         dataType: type,
         comment,
-      };
+      });
     });
-    setFields(columnFields || []);
+    setFields(Array.from(fieldMap.values()));
   };
 
   const formatterIdentifiers = (identifiersList: any[] = []) => {