topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Implement SemanticModeller through SPI (#1865)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-10-30 23:32:38 +08:00
committed by GitHub
parent fa75a13888
commit 838745d415
8 changed files with 77 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldSchema.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
import lombok.Data;
@Data
public class FieldSchema {
public class ColumnSchema {
private String columnName;

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java
View File

@@ -14,11 +14,11 @@ public class ModelSchema {
private String description;
private List<FieldSchema> filedSchemas;
private List<ColumnSchema> columnSchemas;
@JsonIgnore
public FieldSchema getFieldByName(String columnName) {
for (FieldSchema fieldSchema : filedSchemas) {
public ColumnSchema getColumnByName(String columnName) {
for (ColumnSchema fieldSchema : columnSchemas) {
if (fieldSchema.getColumnName().equalsIgnoreCase(columnName)) {
return fieldSchema;
}

35
headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java Normal file
View File

@@ -0,0 +1,35 @@
package com.tencent.supersonic.headless.server.modeller;
import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.api.pojo.DbSchema;
import com.tencent.supersonic.headless.api.pojo.ModelSchema;
import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class RuleSemanticModeller implements SemanticModeller {
@Override
public ModelSchema build(DbSchema dbSchema, List<DbSchema> dbSchemas,
ModelBuildReq modelBuildReq) {
ModelSchema modelSchema = new ModelSchema();
List<ColumnSchema> columnSchemas =
dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
modelSchema.setColumnSchemas(columnSchemas);
return modelSchema;
}
private ColumnSchema convert(DBColumn dbColumn) {
ColumnSchema columnSchema = new ColumnSchema();
columnSchema.setName(dbColumn.getComment());
columnSchema.setColumnName(dbColumn.getColumnName());
columnSchema.setComment(dbColumn.getComment());
columnSchema.setDataType(dbColumn.getDataType());
return columnSchema;
}
}

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.headless.server.modeller;
import com.tencent.supersonic.headless.api.pojo.DbSchema;
import com.tencent.supersonic.headless.api.pojo.ModelSchema;
import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
import java.util.List;
public interface SemanticModeller {
ModelSchema build(DbSchema dbSchema, List<DbSchema> otherDbSchema, ModelBuildReq modelBuildReq);
}

41
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -9,12 +9,10 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.service.ChatModelService;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.api.pojo.DbSchema;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.FieldSchema;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -34,6 +32,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.server.modeller.SemanticModeller;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -46,6 +45,7 @@ import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.utils.ModelConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
@@ -94,8 +94,6 @@ public class ModelServiceImpl implements ModelService {
private DateInfoRepository dateInfoRepository;
private ChatModelService chatModelService;
private ModelRelaService modelRelaService;
ExecutorService executor =
@@ -104,8 +102,7 @@ public class ModelServiceImpl implements ModelService {
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
@Lazy DimensionService dimensionService, @Lazy MetricService metricService,
DomainService domainService, UserService userService, DataSetService dataSetService,
DateInfoRepository dateInfoRepository, ChatModelService chatModelService,
ModelRelaService modelRelaService) {
DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService) {
this.modelRepository = modelRepository;
this.databaseService = databaseService;
this.dimensionService = dimensionService;
@@ -114,7 +111,6 @@ public class ModelServiceImpl implements ModelService {
this.userService = userService;
this.dataSetService = dataSetService;
this.dateInfoRepository = dateInfoRepository;
this.chatModelService = chatModelService;
this.modelRelaService = modelRelaService;
}
@@ -216,19 +212,19 @@ public class ModelServiceImpl implements ModelService {
throws SQLException {
List<DbSchema> dbSchemas = getDbSchemes(modelBuildReq);
Map<String, ModelSchema> modelSchemaMap = new ConcurrentHashMap<>();
CompletableFuture.allOf(dbSchemas
.stream().map(dbSchema -> CompletableFuture
.runAsync(() -> doBuild(dbSchema, modelSchemaMap), executor))
CompletableFuture.allOf(dbSchemas.stream()
.map(dbSchema -> CompletableFuture.runAsync(
() -> doBuild(modelBuildReq, dbSchema, dbSchemas, modelSchemaMap),
executor))
.toArray(CompletableFuture[]::new)).join();
return modelSchemaMap;
}
private void doBuild(DbSchema dbSchema, Map<String, ModelSchema> modelSchemaMap) {
ModelSchema modelSchema = new ModelSchema();
List<FieldSchema> fieldSchemas =
dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
modelSchema.setFiledSchemas(fieldSchemas);
modelSchemaMap.put(dbSchema.getTable(), modelSchema);
private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List<DbSchema> dbSchemas,
Map<String, ModelSchema> modelSchemaMap) {
SemanticModeller semanticModeller = ComponentFactory.getSemanticModeller();
ModelSchema modelSchema = semanticModeller.build(curSchema, dbSchemas, modelBuildReq);
modelSchemaMap.put(curSchema.getTable(), modelSchema);
}
private List<DbSchema> getDbSchemes(ModelBuildReq modelBuildReq) throws SQLException {
@@ -237,9 +233,9 @@ public class ModelServiceImpl implements ModelService {
}
private List<DbSchema> convert(Map<String, List<DBColumn>> dbColumnMap,
ModelBuildReq modelSchemaReq) {
ModelBuildReq modelBuildReq) {
return dbColumnMap.keySet().stream()
.map(key -> convert(modelSchemaReq, key, dbColumnMap.get(key)))
.map(key -> convert(modelBuildReq, key, dbColumnMap.get(key)))
.collect(Collectors.toList());
}
@@ -252,15 +248,6 @@ public class ModelServiceImpl implements ModelService {
return dbSchema;
}
private FieldSchema convert(DBColumn dbColumn) {
FieldSchema fieldSchema = new FieldSchema();
fieldSchema.setName(dbColumn.getComment());
fieldSchema.setColumnName(dbColumn.getColumnName());
fieldSchema.setComment(dbColumn.getComment());
fieldSchema.setDataType(dbColumn.getDataType());
return fieldSchema;
}
private void batchCreateDimension(ModelDO modelDO, User user) throws Exception {
List<DimensionReq> dimensionReqs = ModelConverter.convertDimensionList(modelDO);
dimensionService.createDimensionBatch(dimensionReqs, user);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java
View File

@@ -4,6 +4,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.chat.corrector.SemanticCorrector;
import com.tencent.supersonic.headless.chat.mapper.SchemaMapper;
import com.tencent.supersonic.headless.chat.parser.SemanticParser;
import com.tencent.supersonic.headless.server.modeller.SemanticModeller;
import com.tencent.supersonic.headless.server.processor.ResultProcessor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -19,6 +20,7 @@ public class ComponentFactory {
private static List<SchemaMapper> schemaMappers = new ArrayList<>();
private static List<SemanticParser> semanticParsers = new ArrayList<>();
private static List<SemanticCorrector> semanticCorrectors = new ArrayList<>();
private static SemanticModeller semanticModeller;
public static List<ResultProcessor> getResultProcessors() {
return CollectionUtils.isEmpty(resultProcessors)
@@ -43,6 +45,10 @@ public class ComponentFactory {
: semanticCorrectors;
}
public static SemanticModeller getSemanticModeller() {
return semanticModeller == null ? init(SemanticModeller.class) : semanticModeller;
}
public static <T> T getBean(String name, Class<T> tClass) {
return ContextUtils.getContext().getBean(name, tClass);
}

4
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.service.ChatModelService;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.Identify;
@@ -76,11 +75,10 @@ class ModelServiceImplTest {
UserService userService = Mockito.mock(UserService.class);
DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
DataSetService viewService = Mockito.mock(DataSetService.class);
ChatModelService chatModelService = Mockito.mock(ChatModelService.class);
ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class);
return new ModelServiceImpl(modelRepository, databaseService, dimensionService,
metricService, domainService, userService, viewService, dateInfoRepository,
chatModelService, modelRelaService);
modelRelaService);
}
private ModelReq mockModelReq() {

3
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -50,6 +50,9 @@ com.tencent.supersonic.headless.server.processor.ResultProcessor=\
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\
com.tencent.supersonic.headless.server.processor.EntityInfoProcessor
com.tencent.supersonic.headless.server.modeller.SemanticModeller=\
com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller
### chat-server SPIs
com.tencent.supersonic.chat.server.parser.ChatQueryParser=\