(improvement)(Headless) Refactor the SemanticModeller to rule first and then llm, and automatically infer field types in the rule method. (#1900)

Co-authored-by: lxwcodemonkey

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/LLMSemanticModeller.java

@@ -45,7 +45,7 @@ public class LLMSemanticModeller implements SemanticModeller {
             + "\n2. Create a Chinese name for the field and categorize the field into one of the following five types:"
             + "\n   primary_key: This is a unique identifier for a record row in a database."
             + "\n   foreign_key: This is a key in a database whose value is derived from the primary key of another table."
-            + "\n   data_time: This represents the time when data is generated in the data warehouse."
+            + "\n   partition_time: This represents the time when data is generated in the data warehouse."
             + "\n   dimension: Usually a string type, used for grouping and filtering data. No need to generate aggregate functions"
             + "\n   measure: Usually a numeric type, used to quantify data from a certain evaluative perspective. "
             + "              Also, you need to generate aggregate functions(Eg: MAX, MIN, AVG, SUM, COUNT) for the measure type. "
@@ -66,22 +66,23 @@ public class LLMSemanticModeller implements SemanticModeller {
     }
 
     @Override
-    public ModelSchema build(DbSchema dbSchema, List<DbSchema> dbSchemas,
+    public void build(DbSchema dbSchema, List<DbSchema> dbSchemas, ModelSchema modelSchema,
             ModelBuildReq modelBuildReq) {
+        if (!modelBuildReq.isBuildByLLM()) {
+            return;
+        }
         Optional<ChatApp> chatApp = ChatAppManager.getApp(APP_KEY);
         if (!chatApp.isPresent() || !chatApp.get().isEnable()) {
-            return null;
+            return;
         }
         List<DbSchema> otherDbSchema = getOtherDbSchema(dbSchema, dbSchemas);
         ModelSchemaExtractor extractor =
                 AiServices.create(ModelSchemaExtractor.class, getChatModel(modelBuildReq));
         Prompt prompt = generatePrompt(dbSchema, otherDbSchema, chatApp.get());
-        ModelSchema modelSchema =
-                extractor.generateModelSchema(prompt.toUserMessage().singleText());
+        modelSchema = extractor.generateModelSchema(prompt.toUserMessage().singleText());
         log.info("dbSchema:  {}\n otherRelatedDBSchema:{}\n modelSchema: {}",
                 JsonUtil.toString(dbSchema), JsonUtil.toString(otherDbSchema),
                 JsonUtil.toString(modelSchema));
-        return modelSchema;
     }
 
     private List<DbSchema> getOtherDbSchema(DbSchema curSchema, List<DbSchema> dbSchemas) {
@@ -113,7 +114,7 @@ public class LLMSemanticModeller implements SemanticModeller {
         Environment environment = ContextUtils.getBean(Environment.class);
         String enableExemplarLoading =
                 environment.getProperty("s2.model.building.exemplars.enabled");
-        if (Boolean.TRUE.equals(Boolean.parseBoolean(enableExemplarLoading))) {
+        if (Boolean.FALSE.equals(Boolean.parseBoolean(enableExemplarLoading))) {
             log.info("Not enable load model-building exemplars");
             return "";
         }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java

@@ -14,13 +14,11 @@ import java.util.stream.Collectors;
 public class RuleSemanticModeller implements SemanticModeller {
 
     @Override
-    public ModelSchema build(DbSchema dbSchema, List<DbSchema> dbSchemas,
+    public void build(DbSchema dbSchema, List<DbSchema> dbSchemas, ModelSchema modelSchema,
             ModelBuildReq modelBuildReq) {
-        ModelSchema modelSchema = new ModelSchema();
         List<ColumnSchema> columnSchemas =
                 dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
         modelSchema.setColumnSchemas(columnSchemas);
-        return modelSchema;
     }
 
     private ColumnSchema convert(DBColumn dbColumn) {
@@ -29,6 +27,7 @@ public class RuleSemanticModeller implements SemanticModeller {
         columnSchema.setColumnName(dbColumn.getColumnName());
         columnSchema.setComment(dbColumn.getComment());
         columnSchema.setDataType(dbColumn.getDataType());
+        columnSchema.setFiledType(dbColumn.getFieldType());
         return columnSchema;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java

@@ -9,6 +9,7 @@ import java.util.List;
 
 public interface SemanticModeller {
 
-    ModelSchema build(DbSchema dbSchema, List<DbSchema> otherDbSchema, ModelBuildReq modelBuildReq);
+    void build(DbSchema dbSchema, List<DbSchema> otherDbSchema, ModelSchema modelSchema,
+            ModelBuildReq modelBuildReq);
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -222,8 +222,11 @@ public class ModelServiceImpl implements ModelService {
 
     private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List<DbSchema> dbSchemas,
             Map<String, ModelSchema> modelSchemaMap) {
-        SemanticModeller semanticModeller = CoreComponentFactory.getSemanticModeller();
-        ModelSchema modelSchema = semanticModeller.build(curSchema, dbSchemas, modelBuildReq);
+        ModelSchema modelSchema = new ModelSchema();
+        List<SemanticModeller> semanticModellers = CoreComponentFactory.getSemanticModellers();
+        for (SemanticModeller semanticModeller : semanticModellers) {
+            semanticModeller.build(curSchema, dbSchemas, modelSchema, modelBuildReq);
+        }
         modelSchemaMap.put(curSchema.getTable(), modelSchema);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/CoreComponentFactory.java

@@ -4,15 +4,26 @@ import com.tencent.supersonic.headless.chat.utils.ComponentFactory;
 import com.tencent.supersonic.headless.server.modeller.SemanticModeller;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * QueryConverter QueryOptimizer QueryExecutor object factory
  */
 @Slf4j
 public class CoreComponentFactory extends ComponentFactory {
 
-    private static SemanticModeller semanticModeller;
+    private static List<SemanticModeller> semanticModellers = new ArrayList<>();
 
-    public static SemanticModeller getSemanticModeller() {
-        return semanticModeller == null ? init(SemanticModeller.class) : semanticModeller;
+    public static List<SemanticModeller> getSemanticModellers() {
+        if (semanticModellers.isEmpty()) {
+            initSemanticModellers();
+        }
+        return semanticModellers;
     }
+
+    private static void initSemanticModellers() {
+        init(SemanticModeller.class, semanticModellers);
+    }
+
 }