(improvement)(headless) Refactor HeadlessModel-related code, remove modelSchemaResps. (#647)

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java

@@ -11,14 +11,14 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
-import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
+import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
-import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
+import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
 import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
@@ -55,21 +55,21 @@ import java.util.stream.Collectors;
 public class HeadlessSchemaManager {
 
     @Autowired
-    private LoadingCache<String, HeadlessModel> loadingCache;
+    private LoadingCache<String, SemanticModel> loadingCache;
     private final Catalog catalog;
 
     public HeadlessSchemaManager(Catalog catalog) {
         this.catalog = catalog;
     }
 
-    public HeadlessModel reload(String rootPath) {
-        HeadlessModel headlessModel = new HeadlessModel();
-        headlessModel.setRootPath(rootPath);
+    public SemanticModel reload(String rootPath) {
+        SemanticModel semanticModel = new SemanticModel();
+        semanticModel.setRootPath(rootPath);
         Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim()))
                 .collect(Collectors.toSet());
         if (modelIds.isEmpty()) {
             log.error("get modelIds empty {}", rootPath);
-            return headlessModel;
+            return semanticModel;
         }
         Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
         List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
@@ -77,33 +77,33 @@ public class HeadlessSchemaManager {
         Map<Long, String> modelIdName = new HashMap<>();
         catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
         DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
-        headlessModel.setDatabaseResp(databaseResp);
+        semanticModel.setDatabaseResp(databaseResp);
         List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
         if (!CollectionUtils.isEmpty(modelRelas)) {
-            headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
+            semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
         }
         if (!dataModelYamlTpls.isEmpty()) {
             Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource)
                     .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
-            headlessModel.setDatasourceMap(dataSourceMap);
+            semanticModel.setDatasourceMap(dataSourceMap);
         }
         if (!dimensionYamlTpls.isEmpty()) {
             Map<String, List<Dimension>> dimensionMap = new HashMap<>();
             for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
                 dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
             }
-            headlessModel.setDimensionMap(dimensionMap);
+            semanticModel.setDimensionMap(dimensionMap);
         }
         if (!metricYamlTpls.isEmpty()) {
-            headlessModel.setMetrics(getMetrics(metricYamlTpls));
+            semanticModel.setMetrics(getMetrics(metricYamlTpls));
         }
-        return headlessModel;
+        return semanticModel;
     }
 
     //private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
-    public HeadlessModel get(String rootPath) throws Exception {
+    public SemanticModel get(String rootPath) throws Exception {
         rootPath = formatKey(rootPath);
-        HeadlessModel schema = loadingCache.get(rootPath);
+        SemanticModel schema = loadingCache.get(rootPath);
         if (schema == null) {
             return null;
         }
@@ -182,6 +182,8 @@ public class HeadlessSchemaManager {
             dimension.setExpr(dimensionYamlTpl.getExpr());
             dimension.setName(dimensionYamlTpl.getName());
             dimension.setOwners(dimensionYamlTpl.getOwners());
+            dimension.setBizName(dimensionYamlTpl.getBizName());
+            dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
             if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
                 dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
             }
@@ -235,13 +237,13 @@ public class HeadlessSchemaManager {
         return joinRelations;
     }
 
-    public static void update(HeadlessSchema schema, List<Metric> metric) throws Exception {
+    public static void update(SemanticSchema schema, List<Metric> metric) throws Exception {
         if (schema != null) {
             updateMetric(metric, schema.getMetrics());
         }
     }
 
-    public static void update(HeadlessSchema schema, DataSource datasourceYamlTpl) throws Exception {
+    public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception {
         if (schema != null) {
             String dataSourceName = datasourceYamlTpl.getName();
             Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
@@ -254,7 +256,7 @@ public class HeadlessSchemaManager {
         }
     }
 
-    public static void update(HeadlessSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
+    public static void update(SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
             throws Exception {
         if (schema != null) {
             Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema.getDimension().entrySet()
@@ -320,15 +322,15 @@ public class HeadlessSchemaManager {
         private Integer maximumSize = 1000;
 
         @Bean
-        public LoadingCache<String, HeadlessModel> getCache() {
-            LoadingCache<String, HeadlessModel> cache
+        public LoadingCache<String, SemanticModel> getCache() {
+            LoadingCache<String, SemanticModel> cache
                     = CacheBuilder.newBuilder()
                     .expireAfterWrite(saveMinutes, TimeUnit.MINUTES)
                     .initialCapacity(10)
                     .maximumSize(maximumSize).build(
-                            new CacheLoader<String, HeadlessModel>() {
+                            new CacheLoader<String, SemanticModel>() {
                                 @Override
-                                public HeadlessModel load(String key) {
+                                public SemanticModel load(String key) {
                                     log.info("load SemanticSchema [{}]", key);
                                     return HeadlessSchemaManager.this.reload(key);
                                 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java

@@ -4,39 +4,34 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
 import com.tencent.supersonic.headless.api.request.MetricQueryReq;
 import com.tencent.supersonic.headless.api.request.ParseSqlReq;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.QueryParser;
-import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
+import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.ComponentFactory;
 import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
-import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
 import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
 import com.tencent.supersonic.headless.server.utils.QueryUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
-import java.util.List;
 
 @Slf4j
 @Component
 public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
 
     private final QueryParser queryParser;
-    private final Catalog catalog;
     private final QueryUtils queryUtils;
     private final QueryStructUtils queryStructUtils;
     private final HeadlessSchemaManager headlessSchemaManager;
 
-    public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog,
+    public HeadlessQueryEngineImpl(QueryParser queryParser,
             QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager,
             QueryStructUtils queryStructUtils) {
         this.queryParser = queryParser;
-        this.catalog = catalog;
         this.queryUtils = queryUtils;
         this.headlessSchemaManager = headlessSchemaManager;
         this.queryStructUtils = queryStructUtils;
@@ -57,7 +52,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
 
     public QueryStatement plan(QueryStatement queryStatement) throws Exception {
         queryStatement.setEnableOptimize(queryUtils.enableOptimize());
-        queryStatement.setHeadlessModel(getHeadLessModel(queryStatement));
+        queryStatement.setSemanticModel(getSemanticModel(queryStatement));
         queryStatement = queryParser.logicSql(queryStatement);
         queryUtils.checkSqlParse(queryStatement);
         queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
@@ -82,31 +77,31 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
     }
 
     @Override
-    public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) {
+    public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
         QueryStatement queryStatement = new QueryStatement();
         queryStatement.setQueryStructReq(queryStructCmd);
         queryStatement.setParseSqlReq(sqlCommend);
         queryStatement.setSql(sqlCommend.getSql());
         queryStatement.setIsS2SQL(true);
+        queryStatement.setSemanticModel(getSemanticModel(queryStatement));
         return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
     }
 
-    public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) {
+    public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
         QueryStatement queryStatement = new QueryStatement();
         queryStatement.setQueryStructReq(queryStructCmd);
         queryStatement.setMetricReq(metricCommand);
         queryStatement.setIsS2SQL(false);
+        queryStatement.setSemanticModel(getSemanticModel(queryStatement));
         return queryParser.parser(queryStatement);
     }
 
-    private HeadlessModel getHeadLessModel(QueryStatement queryStatement) throws Exception {
+    private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
         QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
-        HeadlessModel headlessModel = headlessSchemaManager.get(queryStructReq.getModelIdStr());
+        SemanticModel semanticModel = headlessSchemaManager.get(queryStructReq.getModelIdStr());
         ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq);
-        headlessModel.setDataDate(itemDateResp);
-        List<ModelSchemaResp> modelSchemaResps = catalog.getModelSchema(queryStructReq.getModelIds());
-        headlessModel.setModelSchemaResps(modelSchemaResps);
-        return headlessModel;
+        semanticModel.setDataDate(itemDateResp);
+        return semanticModel;
     }
 
 }

headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.enums.AggOption;
 import com.tencent.supersonic.headless.api.request.MetricQueryReq;
 import com.tencent.supersonic.headless.api.response.SqlParserResp;
 import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
-import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
+import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
@@ -26,20 +26,20 @@ import java.util.Map;
 @Slf4j
 class HeadlessParserServiceTest {
 
-    private static Map<String, HeadlessSchema> headlessSchemaMap = new HashMap<>();
+    private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>();
 
-    public static SqlParserResp parser(HeadlessSchema headlessSchema, MetricQueryReq metricCommand, boolean isAgg) {
+    public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricCommand, boolean isAgg) {
         SqlParserResp sqlParser = new SqlParserResp();
         if (metricCommand.getRootPath().isEmpty()) {
             sqlParser.setErrMsg("rootPath empty");
             return sqlParser;
         }
         try {
-            if (headlessSchema == null) {
+            if (semanticSchema == null) {
                 sqlParser.setErrMsg("headlessSchema not found");
                 return sqlParser;
             }
-            AggPlanner aggBuilder = new AggPlanner(headlessSchema);
+            AggPlanner aggBuilder = new AggPlanner(semanticSchema);
             QueryStatement queryStatement = new QueryStatement();
             queryStatement.setMetricReq(metricCommand);
             aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
@@ -122,9 +122,9 @@ class HeadlessParserServiceTest {
         identifies.add(identify);
         datasource.setIdentifiers(identifies);
 
-        HeadlessSchema headlessSchema = HeadlessSchema.newBuilder("s2").build();
+        SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build();
 
-        HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getDatasource(datasource));
+        HeadlessSchemaManager.update(semanticSchema, HeadlessSchemaManager.getDatasource(datasource));
 
         DimensionYamlTpl dimension1 = new DimensionYamlTpl();
         dimension1.setExpr("page");
@@ -133,7 +133,7 @@ class HeadlessParserServiceTest {
         List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
         dimensionYamlTpls.add(dimension1);
 
-        HeadlessSchemaManager.update(headlessSchema, "s2_pv_uv_statis",
+        HeadlessSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
                 HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
 
         MetricYamlTpl metric1 = new MetricYamlTpl();
@@ -174,9 +174,9 @@ class HeadlessParserServiceTest {
         List<ColumnOrder> orders = new ArrayList<>();
         orders.add(ColumnOrder.buildDesc("sys_imp_date"));
         metricCommand.setOrder(orders);
-        System.out.println(parser(headlessSchema, metricCommand, true));
+        System.out.println(parser(semanticSchema, metricCommand, true));
 
-        addDepartment(headlessSchema);
+        addDepartment(semanticSchema);
 
         MetricQueryReq metricCommand2 = new MetricQueryReq();
         metricCommand2.setRootPath("s2");
@@ -189,12 +189,12 @@ class HeadlessParserServiceTest {
         List<ColumnOrder> orders2 = new ArrayList<>();
         orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
         metricCommand2.setOrder(orders2);
-        System.out.println(parser(headlessSchema, metricCommand2, true));
+        System.out.println(parser(semanticSchema, metricCommand2, true));
 
 
     }
 
-    private static void addDepartment(HeadlessSchema headlessSchema) {
+    private static void addDepartment(SemanticSchema semanticSchema) {
         DataModelYamlTpl datasource = new DataModelYamlTpl();
         datasource.setName("user_department");
         datasource.setSourceId(1L);
@@ -240,7 +240,7 @@ class HeadlessParserServiceTest {
         identifies.add(identify);
         datasource.setIdentifiers(identifies);
 
-        headlessSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource));
+        semanticSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource));
 
         DimensionYamlTpl dimension1 = new DimensionYamlTpl();
         dimension1.setExpr("department");
@@ -249,7 +249,7 @@ class HeadlessParserServiceTest {
         List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
         dimensionYamlTpls.add(dimension1);
 
-        headlessSchema.getDimension()
+        semanticSchema.getDimension()
                 .put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
     }
 }