diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java similarity index 93% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldSchema.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java index 2fcb41b6e..79670baf0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.FieldType; import lombok.Data; @Data -public class FieldSchema { +public class ColumnSchema { private String columnName; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java index 781d10bd7..d1429cad2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java @@ -14,11 +14,11 @@ public class ModelSchema { private String description; - private List filedSchemas; + private List columnSchemas; @JsonIgnore - public FieldSchema getFieldByName(String columnName) { - for (FieldSchema fieldSchema : filedSchemas) { + public ColumnSchema getColumnByName(String columnName) { + for (ColumnSchema fieldSchema : columnSchemas) { if (fieldSchema.getColumnName().equalsIgnoreCase(columnName)) { return fieldSchema; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java new file mode 100644 index 000000000..c008eabd3 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java @@ -0,0 +1,35 @@ +package com.tencent.supersonic.headless.server.modeller; + +import com.tencent.supersonic.headless.api.pojo.ColumnSchema; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.api.pojo.DbSchema; +import com.tencent.supersonic.headless.api.pojo.ModelSchema; +import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class RuleSemanticModeller implements SemanticModeller { + + @Override + public ModelSchema build(DbSchema dbSchema, List dbSchemas, + ModelBuildReq modelBuildReq) { + ModelSchema modelSchema = new ModelSchema(); + List columnSchemas = + dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList()); + modelSchema.setColumnSchemas(columnSchemas); + return modelSchema; + } + + private ColumnSchema convert(DBColumn dbColumn) { + ColumnSchema columnSchema = new ColumnSchema(); + columnSchema.setName(dbColumn.getComment()); + columnSchema.setColumnName(dbColumn.getColumnName()); + columnSchema.setComment(dbColumn.getComment()); + columnSchema.setDataType(dbColumn.getDataType()); + return columnSchema; + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java new file mode 100644 index 000000000..e1a287581 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.headless.server.modeller; + + +import com.tencent.supersonic.headless.api.pojo.DbSchema; +import com.tencent.supersonic.headless.api.pojo.ModelSchema; +import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq; + +import java.util.List; + +public interface SemanticModeller { + + ModelSchema build(DbSchema dbSchema, List otherDbSchema, ModelBuildReq modelBuildReq); + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index 7a40ea6b1..c4d0e5fc1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -9,12 +9,10 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; -import com.tencent.supersonic.common.service.ChatModelService; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.DbSchema; import com.tencent.supersonic.headless.api.pojo.Dim; -import com.tencent.supersonic.headless.api.pojo.FieldSchema; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.Measure; @@ -34,6 +32,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; +import com.tencent.supersonic.headless.server.modeller.SemanticModeller; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; @@ -46,6 +45,7 @@ import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.utils.ComponentFactory; import com.tencent.supersonic.headless.server.utils.ModelConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; @@ -94,8 +94,6 @@ public class ModelServiceImpl implements ModelService { private DateInfoRepository dateInfoRepository; - private ChatModelService chatModelService; - private ModelRelaService modelRelaService; ExecutorService executor = @@ -104,8 +102,7 @@ public class ModelServiceImpl implements ModelService { public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService, @Lazy DimensionService dimensionService, @Lazy MetricService metricService, DomainService domainService, UserService userService, DataSetService dataSetService, - DateInfoRepository dateInfoRepository, ChatModelService chatModelService, - ModelRelaService modelRelaService) { + DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService) { this.modelRepository = modelRepository; this.databaseService = databaseService; this.dimensionService = dimensionService; @@ -114,7 +111,6 @@ public class ModelServiceImpl implements ModelService { this.userService = userService; this.dataSetService = dataSetService; this.dateInfoRepository = dateInfoRepository; - this.chatModelService = chatModelService; this.modelRelaService = modelRelaService; } @@ -216,19 +212,19 @@ public class ModelServiceImpl implements ModelService { throws SQLException { List dbSchemas = getDbSchemes(modelBuildReq); Map modelSchemaMap = new ConcurrentHashMap<>(); - CompletableFuture.allOf(dbSchemas - .stream().map(dbSchema -> CompletableFuture - .runAsync(() -> doBuild(dbSchema, modelSchemaMap), executor)) + CompletableFuture.allOf(dbSchemas.stream() + .map(dbSchema -> CompletableFuture.runAsync( + () -> doBuild(modelBuildReq, dbSchema, dbSchemas, modelSchemaMap), + executor)) .toArray(CompletableFuture[]::new)).join(); return modelSchemaMap; } - private void doBuild(DbSchema dbSchema, Map modelSchemaMap) { - ModelSchema modelSchema = new ModelSchema(); - List fieldSchemas = - dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList()); - modelSchema.setFiledSchemas(fieldSchemas); - modelSchemaMap.put(dbSchema.getTable(), modelSchema); + private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List dbSchemas, + Map modelSchemaMap) { + SemanticModeller semanticModeller = ComponentFactory.getSemanticModeller(); + ModelSchema modelSchema = semanticModeller.build(curSchema, dbSchemas, modelBuildReq); + modelSchemaMap.put(curSchema.getTable(), modelSchema); } private List getDbSchemes(ModelBuildReq modelBuildReq) throws SQLException { @@ -237,9 +233,9 @@ public class ModelServiceImpl implements ModelService { } private List convert(Map> dbColumnMap, - ModelBuildReq modelSchemaReq) { + ModelBuildReq modelBuildReq) { return dbColumnMap.keySet().stream() - .map(key -> convert(modelSchemaReq, key, dbColumnMap.get(key))) + .map(key -> convert(modelBuildReq, key, dbColumnMap.get(key))) .collect(Collectors.toList()); } @@ -252,15 +248,6 @@ public class ModelServiceImpl implements ModelService { return dbSchema; } - private FieldSchema convert(DBColumn dbColumn) { - FieldSchema fieldSchema = new FieldSchema(); - fieldSchema.setName(dbColumn.getComment()); - fieldSchema.setColumnName(dbColumn.getColumnName()); - fieldSchema.setComment(dbColumn.getComment()); - fieldSchema.setDataType(dbColumn.getDataType()); - return fieldSchema; - } - private void batchCreateDimension(ModelDO modelDO, User user) throws Exception { List dimensionReqs = ModelConverter.convertDimensionList(modelDO); dimensionService.createDimensionBatch(dimensionReqs, user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java index db3223699..c882d5784 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.chat.corrector.SemanticCorrector; import com.tencent.supersonic.headless.chat.mapper.SchemaMapper; import com.tencent.supersonic.headless.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.server.modeller.SemanticModeller; import com.tencent.supersonic.headless.server.processor.ResultProcessor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -19,6 +20,7 @@ public class ComponentFactory { private static List schemaMappers = new ArrayList<>(); private static List semanticParsers = new ArrayList<>(); private static List semanticCorrectors = new ArrayList<>(); + private static SemanticModeller semanticModeller; public static List getResultProcessors() { return CollectionUtils.isEmpty(resultProcessors) @@ -43,6 +45,10 @@ public class ComponentFactory { : semanticCorrectors; } + public static SemanticModeller getSemanticModeller() { + return semanticModeller == null ? init(SemanticModeller.class) : semanticModeller; + } + public static T getBean(String name, Class tClass) { return ContextUtils.getContext().getBean(name, tClass); } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java index 74ad8c456..12814828c 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java @@ -5,7 +5,6 @@ import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.service.ChatModelService; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.Identify; @@ -76,11 +75,10 @@ class ModelServiceImplTest { UserService userService = Mockito.mock(UserService.class); DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); DataSetService viewService = Mockito.mock(DataSetService.class); - ChatModelService chatModelService = Mockito.mock(ChatModelService.class); ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class); return new ModelServiceImpl(modelRepository, databaseService, dimensionService, metricService, domainService, userService, viewService, dateInfoRepository, - chatModelService, modelRelaService); + modelRelaService); } private ModelReq mockModelReq() { diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 2a5caf958..cdecc8954 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -50,6 +50,9 @@ com.tencent.supersonic.headless.server.processor.ResultProcessor=\ com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\ com.tencent.supersonic.headless.server.processor.EntityInfoProcessor +com.tencent.supersonic.headless.server.modeller.SemanticModeller=\ + com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller + ### chat-server SPIs com.tencent.supersonic.chat.server.parser.ChatQueryParser=\