(improvement)(headless)(chat) Add views and adapt chat and headless (#700)

* (improvement)(headless)(chat) Add views and adapt chat and headless

---------

Co-authored-by: jolunoluo

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java

@@ -11,16 +11,17 @@ import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.ComponentFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 @Component
 @Slf4j
 @Primary
@@ -109,10 +110,6 @@ public class DefaultQueryParser implements QueryParser {
     public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
         MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
         log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
-        if (metricQueryReq.getRootPath().isEmpty()) {
-            queryStatement.setErrMsg("rootPath empty");
-            return queryStatement;
-        }
         try {
             return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
         } catch (Exception e) {
@@ -129,13 +126,12 @@ public class DefaultQueryParser implements QueryParser {
         metricReq.setDimensions(metricTable.getDimensions());
         metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
         metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
-        metricReq.setRootPath(parseSqlReq.getRootPath());
         QueryStatement tableSql = new QueryStatement();
         tableSql.setIsS2SQL(false);
         tableSql.setMetricReq(metricReq);
         tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
         tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
-        tableSql.setModelIds(queryStatement.getModelIds());
+        tableSql.setViewId(queryStatement.getViewId());
         tableSql.setSemanticModel(queryStatement.getSemanticModel());
         if (isSingleMetricTable) {
             tableSql.setViewSql(parseSqlReq.getSql());

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java

@@ -6,13 +6,14 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
 import com.tencent.supersonic.headless.core.parser.SqlParser;
 import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
-import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
 import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions;
+import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
-import java.util.Objects;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
+import java.util.Objects;
+
 /**
  * the calcite parse implements
  */
@@ -50,7 +51,7 @@ public class CalciteSqlParser implements SqlParser {
     }
 
     private SemanticSchema getSemanticSchema(SemanticModel semanticModel, QueryStatement queryStatement) {
-        SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build();
+        SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build();
         semanticSchema.setSemanticModel(semanticModel);
         semanticSchema.setDatasource(semanticModel.getDatasourceMap());
         semanticSchema.setDimension(semanticModel.getDimensionMap());

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java

@@ -1,18 +1,19 @@
 package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
 
 import com.tencent.supersonic.headless.core.pojo.Database;
+import lombok.Data;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-import lombok.Data;
 
 @Data
 public class SemanticModel {
 
-    private String rootPath;
+    private String schemaKey;
     private List<Metric> metrics = new ArrayList<>();
     private Map<String, DataSource> datasourceMap = new HashMap<>();
     private Map<String, List<Dimension>> dimensionMap = new HashMap<>();

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java

@@ -4,11 +4,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.schema;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
 import com.tencent.supersonic.headless.core.parser.calcite.sql.S2SQLSqlValidatorImpl;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
 import org.apache.calcite.jdbc.CalciteSchema;
 import org.apache.calcite.prepare.CalciteCatalogReader;
 import org.apache.calcite.prepare.Prepare;
@@ -19,6 +14,12 @@ import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.sql.validate.ParameterScope;
 import org.apache.calcite.sql.validate.SqlValidatorScope;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
 public class SchemaBuilder {
 
     public static final String MATERIALIZATION_SYS_DB = "SYS";
@@ -30,10 +31,10 @@ public class SchemaBuilder {
     public static SqlValidatorScope getScope(SemanticSchema schema) throws Exception {
         Map<String, RelDataType> nameToTypeMap = new HashMap<>();
         CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false);
-        rootSchema.add(schema.getRootPath(), schema);
+        rootSchema.add(schema.getSchemaKey(), schema);
         Prepare.CatalogReader catalogReader = new CalciteCatalogReader(
                 rootSchema,
-                Collections.singletonList(schema.getRootPath()),
+                Collections.singletonList(schema.getSchemaKey()),
                 Configuration.typeFactory,
                 Configuration.config
         );

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java

@@ -19,7 +19,7 @@ import java.util.Map;
 
 public class SemanticSchema extends AbstractSchema {
 
-    private final String rootPath;
+    private final String schemaKey;
     private final Map<String, Table> tableMap;
 
     private SemanticModel semanticModel = new SemanticModel();
@@ -29,17 +29,17 @@ public class SemanticSchema extends AbstractSchema {
     private RuntimeOptions runtimeOptions;
 
 
-    private SemanticSchema(String rootPath, Map<String, Table> tableMap) {
-        this.rootPath = rootPath;
+    private SemanticSchema(String schemaKey, Map<String, Table> tableMap) {
+        this.schemaKey = schemaKey;
         this.tableMap = tableMap;
     }
 
-    public static Builder newBuilder(String rootPath) {
-        return new Builder(rootPath);
+    public static Builder newBuilder(String schemaKey) {
+        return new Builder(schemaKey);
     }
 
-    public String getRootPath() {
-        return rootPath;
+    public String getSchemaKey() {
+        return schemaKey;
     }
 
     public void setSemanticModel(SemanticModel semanticModel) {
@@ -110,15 +110,15 @@ public class SemanticSchema extends AbstractSchema {
 
     public static final class Builder {
 
-        private final String rootPath;
+        private final String schemaKey;
         private final Map<String, Table> tableMap = new HashMap<>();
 
-        private Builder(String rootPath) {
-            if (rootPath == null || rootPath.isEmpty()) {
+        private Builder(String schemaKey) {
+            if (schemaKey == null) {
                 throw new IllegalArgumentException("Schema name cannot be null or empty");
             }
 
-            this.rootPath = rootPath;
+            this.schemaKey = schemaKey;
         }
 
         public Builder addTable(DataSourceTable table) {
@@ -132,7 +132,7 @@ public class SemanticSchema extends AbstractSchema {
         }
 
         public SemanticSchema build() {
-            return new SemanticSchema(rootPath, tableMap);
+            return new SemanticSchema(schemaKey, tableMap);
         }
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java

@@ -13,14 +13,15 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 /**
  * supplement the QueryStatement when query with custom aggregation method
@@ -49,7 +50,6 @@ public class CalculateAggConverter implements HeadlessConverter {
             return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
         }
         ParseSqlReq sqlCommand = new ParseSqlReq();
-        sqlCommand.setRootPath(queryStructReq.getModelIdStr());
         String metricTableName = "v_metric_tb_tmp";
         MetricTable metricTable = new MetricTable();
         metricTable.setAlias(metricTableName);
@@ -111,7 +111,6 @@ public class CalculateAggConverter implements HeadlessConverter {
                 EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
         sqlCommend.setSql(parseSqlReq.getSql());
         sqlCommend.setTables(parseSqlReq.getTables());
-        sqlCommend.setRootPath(parseSqlReq.getRootPath());
         sqlCommend.setVariables(parseSqlReq.getVariables());
         sqlCommend.setSupportWith(parseSqlReq.isSupportWith());
     }
@@ -137,7 +136,6 @@ public class CalculateAggConverter implements HeadlessConverter {
         check(queryStructReq);
         queryStatement.setEnableOptimize(false);
         ParseSqlReq sqlCommand = new ParseSqlReq();
-        sqlCommand.setRootPath(queryStructReq.getModelIdStr());
         String metricTableName = "v_metric_tb_tmp";
         MetricTable metricTable = new MetricTable();
         metricTable.setAlias(metricTableName);

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java

@@ -63,14 +63,12 @@ public class ParserDefaultConverter implements HeadlessConverter {
         metricQueryReq.setVariables(queryStructReq.getParams().stream()
                 .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
         metricQueryReq.setLimit(queryStructReq.getLimit());
-        String rootPath = queryStructReq.getModelIdStr();
-        metricQueryReq.setRootPath(rootPath);
 
         // support detail query
         if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) {
-            Map<Long, DataSource> dataSourceMap = queryStatement.getSemanticModel().getModelMap();
-            for (Long modelId : queryStructReq.getModelIds()) {
-                String modelBizName = dataSourceMap.get(modelId).getName();
+            Map<Long, DataSource> modelMap = queryStatement.getSemanticModel().getModelMap();
+            for (Long modelId : modelMap.keySet()) {
+                String modelBizName = modelMap.get(modelId).getName();
                 String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName);
                 metricQueryReq.getMetrics().add(internalMetricName);
             }

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java

@@ -3,15 +3,17 @@ package com.tencent.supersonic.headless.core.pojo;
 import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
 import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
-import java.util.List;
 import lombok.Data;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Triple;
 
+import java.util.List;
+
 @Data
 public class QueryStatement {
-
+    private Long viewId;
     private List<Long> modelIds;
     private String sql = "";
     private String sourceId = "";
@@ -30,9 +32,10 @@ public class QueryStatement {
     private String viewSimplifySql = "";
     private Boolean enableLimitWrapper = false;
 
-
     private SemanticModel semanticModel;
 
+    private SemanticSchemaResp semanticSchemaResp;
+
     public boolean isOk() {
         this.ok = "".equals(errMsg) && !"".equals(sql);
         return ok;

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java

@@ -1,12 +1,5 @@
 package com.tencent.supersonic.headless.core.utils;
 
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -17,13 +10,23 @@ import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.StringUtil;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
-import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
+import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
+import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.logging.log4j.util.Strings;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Collections;
@@ -35,14 +38,13 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.logging.log4j.util.Strings;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
+
+import static com.tencent.supersonic.common.pojo.Constants.DAY;
+import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
+import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.MONTH;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.WEEK;
 
 /**
  * tools functions to analyze queryStructReq
@@ -269,19 +271,21 @@ public class SqlGenerateUtils {
         return modelBizName + UNDERLINE + internalMetricNameSuffix;
     }
 
-    public String generateDerivedMetric(final List<MetricResp> metricResps, final Set<String> allFields,
-            final Map<String, Measure> allMeasures, final List<DimensionResp> dimensionResps,
-            final String expression, final MetricDefineType metricDefineType, AggOption aggOption,
-            Set<String> visitedMetric,
-            Set<String> measures,
-            Set<String> dimensions) {
+    public String generateDerivedMetric(final List<MetricSchemaResp> metricResps, final Set<String> allFields,
+                                        final Map<String, Measure> allMeasures,
+                                        final List<DimSchemaResp> dimensionResps,
+                                        final String expression, final MetricDefineType metricDefineType,
+                                        AggOption aggOption,
+                                        Set<String> visitedMetric,
+                                        Set<String> measures,
+                                        Set<String> dimensions) {
         Set<String> fields = SqlParserSelectHelper.getColumnFromExpr(expression);
         if (!CollectionUtils.isEmpty(fields)) {
             Map<String, String> replace = new HashMap<>();
             for (String field : fields) {
                 switch (metricDefineType) {
                     case METRIC:
-                        Optional<MetricResp> metricItem = metricResps.stream()
+                        Optional<MetricSchemaResp> metricItem = metricResps.stream()
                                 .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
                         if (metricItem.isPresent()) {
                             if (visitedMetric.contains(field)) {
@@ -302,7 +306,7 @@ public class SqlGenerateUtils {
                         break;
                     case FIELD:
                         if (allFields.contains(field)) {
-                            Optional<DimensionResp> dimensionItem = dimensionResps.stream()
+                            Optional<DimSchemaResp> dimensionItem = dimensionResps.stream()
                                     .filter(d -> d.getBizName().equals(field)).findFirst();
                             if (dimensionItem.isPresent()) {
                                 dimensions.add(field);