(improvement)(Headless) Implement SemanticModeller through SPI (#1865)

Co-authored-by: lxwcodemonkey

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
 import lombok.Data;
 
 @Data
-public class FieldSchema {
+public class ColumnSchema {
 
     private String columnName;
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java

@@ -14,11 +14,11 @@ public class ModelSchema {
 
     private String description;
 
-    private List<FieldSchema> filedSchemas;
+    private List<ColumnSchema> columnSchemas;
 
     @JsonIgnore
-    public FieldSchema getFieldByName(String columnName) {
+    public ColumnSchema getColumnByName(String columnName) {
-        for (FieldSchema fieldSchema : filedSchemas) {
+        for (ColumnSchema fieldSchema : columnSchemas) {
             if (fieldSchema.getColumnName().equalsIgnoreCase(columnName)) {
                 return fieldSchema;
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java

@@ -0,0 +1,35 @@
+package com.tencent.supersonic.headless.server.modeller;
+
+import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
+import com.tencent.supersonic.headless.api.pojo.DBColumn;
+import com.tencent.supersonic.headless.api.pojo.DbSchema;
+import com.tencent.supersonic.headless.api.pojo.ModelSchema;
+import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class RuleSemanticModeller implements SemanticModeller {
+
+    @Override
+    public ModelSchema build(DbSchema dbSchema, List<DbSchema> dbSchemas,
+            ModelBuildReq modelBuildReq) {
+        ModelSchema modelSchema = new ModelSchema();
+        List<ColumnSchema> columnSchemas =
+                dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
+        modelSchema.setColumnSchemas(columnSchemas);
+        return modelSchema;
+    }
+
+    private ColumnSchema convert(DBColumn dbColumn) {
+        ColumnSchema columnSchema = new ColumnSchema();
+        columnSchema.setName(dbColumn.getComment());
+        columnSchema.setColumnName(dbColumn.getColumnName());
+        columnSchema.setComment(dbColumn.getComment());
+        columnSchema.setDataType(dbColumn.getDataType());
+        return columnSchema;
+    }
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java

@@ -0,0 +1,14 @@
+package com.tencent.supersonic.headless.server.modeller;
+
+
+import com.tencent.supersonic.headless.api.pojo.DbSchema;
+import com.tencent.supersonic.headless.api.pojo.ModelSchema;
+import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
+
+import java.util.List;
+
+public interface SemanticModeller {
+
+    ModelSchema build(DbSchema dbSchema, List<DbSchema> otherDbSchema, ModelBuildReq modelBuildReq);
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -9,12 +9,10 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
-import com.tencent.supersonic.common.service.ChatModelService;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.DbSchema;
 import com.tencent.supersonic.headless.api.pojo.Dim;
-import com.tencent.supersonic.headless.api.pojo.FieldSchema;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -34,6 +32,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
+import com.tencent.supersonic.headless.server.modeller.SemanticModeller;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -46,6 +45,7 @@ import com.tencent.supersonic.headless.server.service.DomainService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelRelaService;
 import com.tencent.supersonic.headless.server.service.ModelService;
+import com.tencent.supersonic.headless.server.utils.ComponentFactory;
 import com.tencent.supersonic.headless.server.utils.ModelConverter;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
 import lombok.extern.slf4j.Slf4j;
@@ -94,8 +94,6 @@ public class ModelServiceImpl implements ModelService {
 
     private DateInfoRepository dateInfoRepository;
 
-    private ChatModelService chatModelService;
-
     private ModelRelaService modelRelaService;
 
     ExecutorService executor =
@@ -104,8 +102,7 @@ public class ModelServiceImpl implements ModelService {
     public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
             @Lazy DimensionService dimensionService, @Lazy MetricService metricService,
             DomainService domainService, UserService userService, DataSetService dataSetService,
-            DateInfoRepository dateInfoRepository, ChatModelService chatModelService,
+            DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService) {
-            ModelRelaService modelRelaService) {
         this.modelRepository = modelRepository;
         this.databaseService = databaseService;
         this.dimensionService = dimensionService;
@@ -114,7 +111,6 @@ public class ModelServiceImpl implements ModelService {
         this.userService = userService;
         this.dataSetService = dataSetService;
         this.dateInfoRepository = dateInfoRepository;
-        this.chatModelService = chatModelService;
         this.modelRelaService = modelRelaService;
     }
 
@@ -216,19 +212,19 @@ public class ModelServiceImpl implements ModelService {
             throws SQLException {
         List<DbSchema> dbSchemas = getDbSchemes(modelBuildReq);
         Map<String, ModelSchema> modelSchemaMap = new ConcurrentHashMap<>();
-        CompletableFuture.allOf(dbSchemas
+        CompletableFuture.allOf(dbSchemas.stream()
-                .stream().map(dbSchema -> CompletableFuture
+                .map(dbSchema -> CompletableFuture.runAsync(
-                        .runAsync(() -> doBuild(dbSchema, modelSchemaMap), executor))
+                        () -> doBuild(modelBuildReq, dbSchema, dbSchemas, modelSchemaMap),
+                        executor))
                 .toArray(CompletableFuture[]::new)).join();
         return modelSchemaMap;
     }
 
-    private void doBuild(DbSchema dbSchema, Map<String, ModelSchema> modelSchemaMap) {
+    private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List<DbSchema> dbSchemas,
-        ModelSchema modelSchema = new ModelSchema();
+            Map<String, ModelSchema> modelSchemaMap) {
-        List<FieldSchema> fieldSchemas =
+        SemanticModeller semanticModeller = ComponentFactory.getSemanticModeller();
-                dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
+        ModelSchema modelSchema = semanticModeller.build(curSchema, dbSchemas, modelBuildReq);
-        modelSchema.setFiledSchemas(fieldSchemas);
+        modelSchemaMap.put(curSchema.getTable(), modelSchema);
-        modelSchemaMap.put(dbSchema.getTable(), modelSchema);
     }
 
     private List<DbSchema> getDbSchemes(ModelBuildReq modelBuildReq) throws SQLException {
@@ -237,9 +233,9 @@ public class ModelServiceImpl implements ModelService {
     }
 
     private List<DbSchema> convert(Map<String, List<DBColumn>> dbColumnMap,
-            ModelBuildReq modelSchemaReq) {
+            ModelBuildReq modelBuildReq) {
         return dbColumnMap.keySet().stream()
-                .map(key -> convert(modelSchemaReq, key, dbColumnMap.get(key)))
+                .map(key -> convert(modelBuildReq, key, dbColumnMap.get(key)))
                 .collect(Collectors.toList());
     }
 
@@ -252,15 +248,6 @@ public class ModelServiceImpl implements ModelService {
         return dbSchema;
     }
 
-    private FieldSchema convert(DBColumn dbColumn) {
-        FieldSchema fieldSchema = new FieldSchema();
-        fieldSchema.setName(dbColumn.getComment());
-        fieldSchema.setColumnName(dbColumn.getColumnName());
-        fieldSchema.setComment(dbColumn.getComment());
-        fieldSchema.setDataType(dbColumn.getDataType());
-        return fieldSchema;
-    }
-
     private void batchCreateDimension(ModelDO modelDO, User user) throws Exception {
         List<DimensionReq> dimensionReqs = ModelConverter.convertDimensionList(modelDO);
         dimensionService.createDimensionBatch(dimensionReqs, user);

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java

@@ -4,6 +4,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.chat.corrector.SemanticCorrector;
 import com.tencent.supersonic.headless.chat.mapper.SchemaMapper;
 import com.tencent.supersonic.headless.chat.parser.SemanticParser;
+import com.tencent.supersonic.headless.server.modeller.SemanticModeller;
 import com.tencent.supersonic.headless.server.processor.ResultProcessor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
@@ -19,6 +20,7 @@ public class ComponentFactory {
     private static List<SchemaMapper> schemaMappers = new ArrayList<>();
     private static List<SemanticParser> semanticParsers = new ArrayList<>();
     private static List<SemanticCorrector> semanticCorrectors = new ArrayList<>();
+    private static SemanticModeller semanticModeller;
 
     public static List<ResultProcessor> getResultProcessors() {
         return CollectionUtils.isEmpty(resultProcessors)
@@ -43,6 +45,10 @@ public class ComponentFactory {
                 : semanticCorrectors;
     }
 
+    public static SemanticModeller getSemanticModeller() {
+        return semanticModeller == null ? init(SemanticModeller.class) : semanticModeller;
+    }
+
     public static <T> T getBean(String name, Class<T> tClass) {
         return ContextUtils.getContext().getBean(name, tClass);
     }

headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java

@@ -5,7 +5,6 @@ import com.tencent.supersonic.auth.api.authentication.service.UserService;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
-import com.tencent.supersonic.common.service.ChatModelService;
 import com.tencent.supersonic.headless.api.pojo.Dim;
 import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.Identify;
@@ -76,11 +75,10 @@ class ModelServiceImplTest {
         UserService userService = Mockito.mock(UserService.class);
         DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
         DataSetService viewService = Mockito.mock(DataSetService.class);
-        ChatModelService chatModelService = Mockito.mock(ChatModelService.class);
         ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class);
         return new ModelServiceImpl(modelRepository, databaseService, dimensionService,
                 metricService, domainService, userService, viewService, dateInfoRepository,
-                chatModelService, modelRelaService);
+                modelRelaService);
     }
 
     private ModelReq mockModelReq() {

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -50,6 +50,9 @@ com.tencent.supersonic.headless.server.processor.ResultProcessor=\
     com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\
     com.tencent.supersonic.headless.server.processor.EntityInfoProcessor
 
+com.tencent.supersonic.headless.server.modeller.SemanticModeller=\
+    com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller
+
 ### chat-server SPIs
 
 com.tencent.supersonic.chat.server.parser.ChatQueryParser=\