(improvement) Move out the datasource and merge the datasource with the model, and adapt the chat module (#423)

Co-authored-by: jolunoluo <jolunoluo@tencent.com>

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java

@@ -1,11 +1,13 @@
 package com.tencent.supersonic.semantic.api.model.pojo;
 
-import java.util.List;
 import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
 
 
 @Data
-public class DatasourceDetail {
+public class ModelDetail {
 
     private String queryType;
 
@@ -19,5 +21,11 @@ public class DatasourceDetail {
 
     private List<Measure> measures;
 
+    public String getSqlQuery() {
+        if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) {
+            sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1);
+        }
+        return sqlQuery;
+    }
 
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceRelaReq.java

@@ -1,23 +0,0 @@
-package com.tencent.supersonic.semantic.api.model.request;
-
-import javax.validation.constraints.NotNull;
-import lombok.Data;
-
-@Data
-public class DatasourceRelaReq {
-
-    private Long id;
-
-    @NotNull(message = "model id cat not be null")
-    private Long modelId;
-
-    @NotNull(message = "datasource id cat not be null")
-    private Long datasourceFrom;
-
-    @NotNull(message = "datasource id cat not be null")
-    private Long datasourceTo;
-
-    @NotNull(message = "join key cat not be null")
-    private String joinKey;
-
-}

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java

@@ -1,56 +0,0 @@
-package com.tencent.supersonic.semantic.api.model.request;
-
-
-import com.google.common.collect.Lists;
-import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
-import com.tencent.supersonic.semantic.api.model.pojo.Dim;
-import com.tencent.supersonic.semantic.api.model.pojo.Identify;
-import com.tencent.supersonic.semantic.api.model.pojo.Measure;
-import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
-import java.util.List;
-import java.util.stream.Collectors;
-import lombok.Data;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.util.CollectionUtils;
-
-
-@Data
-public class DatasourceReq extends SchemaItem {
-
-    private Long databaseId;
-
-    private String queryType;
-
-    private String sqlQuery;
-
-    private String tableQuery;
-
-    private Long modelId;
-
-    private List<Identify> identifiers;
-
-    private List<Dim> dimensions;
-
-    private List<Measure> measures;
-
-    private String filterSql;
-
-
-
-    public List<Dim> getTimeDimension() {
-        if (CollectionUtils.isEmpty(dimensions)) {
-            return Lists.newArrayList();
-        }
-        return dimensions.stream()
-                .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType()))
-                .collect(Collectors.toList());
-    }
-
-    public String getSqlQuery() {
-        if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) {
-            sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1);
-        }
-        return sqlQuery;
-    }
-
-}

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DimensionReq.java

@@ -18,9 +18,6 @@ public class DimensionReq extends SchemaItem {
     @NotNull(message = "expr can not be null")
     private String expr;
 
-
-    private Long datasourceId;
-
     //DATE ID CATEGORY
     private String semanticType = "CATEGORY";
 

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DomainReq.java

@@ -1,14 +1,12 @@
 package com.tencent.supersonic.semantic.api.model.request;
 
 
-
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
+import lombok.Data;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import lombok.Data;
-
 
 @Data
 public class DomainReq extends SchemaItem {
@@ -24,4 +22,22 @@ public class DomainReq extends SchemaItem {
     private List<String> admins = new ArrayList<>();
 
     private List<String> adminOrgs = new ArrayList<>();
+
+    public String getViewer() {
+        return String.join(",", viewers);
+    }
+
+    public String getViewOrg() {
+        return String.join(",", viewOrgs);
+    }
+
+
+    public String getAdmin() {
+        return String.join(",", admins);
+    }
+
+    public String getAdminOrg() {
+        return String.join(",", adminOrgs);
+    }
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java

@@ -1,24 +1,37 @@
 package com.tencent.supersonic.semantic.api.model.request;
 
 
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
+import com.tencent.supersonic.semantic.api.model.pojo.Dim;
 import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
-import com.tencent.supersonic.semantic.api.model.pojo.Entity;
+import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail;
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import lombok.Data;
+import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 
 @Data
 public class ModelReq extends SchemaItem {
 
-    private Long domainId = 0L;
+    private Long databaseId;
 
-    private Integer isOpen = 0;
+    private Long domainId;
+
+    private String filterSql;
+
+    private Integer isOpen;
+
+    private List<DrillDownDimension> drillDownDimensions;
 
     private String alias;
 
+    private ModelDetail modelDetail;
+
     private List<String> viewers = new ArrayList<>();
 
     private List<String> viewOrgs = new ArrayList<>();
@@ -27,9 +40,16 @@ public class ModelReq extends SchemaItem {
 
     private List<String> adminOrgs = new ArrayList<>();
 
-    private Entity entity;
 
-    private List<DrillDownDimension> drillDownDimensions;
+    public List<Dim> getTimeDimension() {
+        if (CollectionUtils.isEmpty(modelDetail.getDimensions())) {
+            return Lists.newArrayList();
+        }
+        return modelDetail.getDimensions().stream()
+                .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType()))
+                .collect(Collectors.toList());
+    }
+
 
     public String getViewer() {
         return String.join(",", viewers);
@@ -48,4 +68,5 @@ public class ModelReq extends SchemaItem {
         return String.join(",", adminOrgs);
     }
 
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceRelaResp.java

@@ -1,27 +0,0 @@
-package com.tencent.supersonic.semantic.api.model.response;
-
-import java.util.Date;
-import lombok.Data;
-
-@Data
-public class DatasourceRelaResp {
-
-    private Long id;
-
-    private Long domainId;
-
-    private Long datasourceFrom;
-
-    private Long datasourceTo;
-
-    private String joinKey;
-
-    private Date createdAt;
-
-    private String createdBy;
-
-    private Date updatedAt;
-
-    private String updatedBy;
-
-}

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java

@@ -1,20 +0,0 @@
-package com.tencent.supersonic.semantic.api.model.response;
-
-import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
-import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
-import lombok.Data;
-
-@Data
-public class DatasourceResp extends SchemaItem {
-
-    private Long modelId;
-
-    private Long databaseId;
-
-    private DatasourceDetail datasourceDetail;
-
-    private String depends;
-
-    private String filterSql;
-
-}

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimSchemaResp.java

@@ -3,10 +3,14 @@ package com.tencent.supersonic.semantic.api.model.response;
 import lombok.Data;
 import lombok.ToString;
 
+import java.util.List;
+
 @Data
 @ToString(callSuper = true)
 public class DimSchemaResp extends DimensionResp {
 
     private Long useCnt = 0L;
 
+    private List<String> entityAlias;
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimensionResp.java

@@ -22,13 +22,11 @@ public class DimensionResp extends SchemaItem {
 
     private String fullPath;
 
-    private Long datasourceId;
+    private String modelName;
 
-    private String datasourceName;
+    private String modelBizName;
 
-    private String datasourceBizName;
-
-    private String datasourceFilterSql;
+    private String modelFilterSql;
     //DATE ID CATEGORY
     private String semanticType;
 

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java

@@ -46,8 +46,6 @@ public class MetricResp extends SchemaItem {
 
     private boolean hasAdminRes = false;
 
-    private String defaultAgg;
-
     public void setTag(String tag) {
         if (StringUtils.isBlank(tag)) {
             tags = Lists.newArrayList();
@@ -80,4 +78,8 @@ public class MetricResp extends SchemaItem {
         }
         return typeParams.getMeasures();
     }
+
+    public String getDefaultAgg() {
+        return typeParams.getMeasures().get(0).getAgg();
+    }
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java

@@ -1,43 +1,66 @@
 package com.tencent.supersonic.semantic.api.model.response;
 
+import com.tencent.supersonic.semantic.api.model.pojo.Identify;
+import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail;
 import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
-import com.tencent.supersonic.semantic.api.model.pojo.Entity;
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import lombok.Data;
-import lombok.ToString;
+import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
 import java.util.List;
 
 @Data
-@ToString
 public class ModelResp extends SchemaItem {
 
     private Long domainId;
 
+    private Long databaseId;
+
+    private ModelDetail modelDetail;
+
+    private String depends;
+
+    private String filterSql;
+
+    private List<String> viewers = new ArrayList<>();
+
+    private List<String> viewOrgs = new ArrayList<>();
+
+    private List<String> admins = new ArrayList<>();
+
+    private List<String> adminOrgs = new ArrayList<>();
+
+    private Integer isOpen;
+
+    private List<DrillDownDimension> drillDownDimensions;
+
     private String alias;
 
-    private List<String> viewers;
-
-    private List<String> viewOrgs;
-
-    private List<String> admins;
-
-    private List<String> adminOrgs;
-
-    private Integer isOpen = 0;
+    private String fullPath;
 
     private Integer dimensionCnt;
 
     private Integer metricCnt;
 
-    private Entity entity;
-
-    private String fullPath;
-
-    private List<DrillDownDimension> drillDownDimensions;
 
     public boolean openToAll() {
         return isOpen != null && isOpen == 1;
     }
 
+    public Identify getPrimaryIdentify() {
+        if (modelDetail == null) {
+            return null;
+        }
+        if (CollectionUtils.isEmpty(modelDetail.getIdentifiers())) {
+            return null;
+        }
+        for (Identify identify : modelDetail.getIdentifiers()) {
+            if (!CollectionUtils.isEmpty(identify.getEntityNames())) {
+                return identify;
+            }
+        }
+        return null;
+    }
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaRelaResp.java

@@ -1,14 +1,15 @@
 package com.tencent.supersonic.semantic.api.model.response;
 
-import java.util.List;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ModelSchemaRelaResp {
 
     private Long domainId;
 
-    private DatasourceResp datasource;
+    private ModelResp model;
 
     private List<MetricResp> metrics;
 

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaResp.java

@@ -1,11 +1,13 @@
 package com.tencent.supersonic.semantic.api.model.response;
 
-import java.util.List;
-
+import com.tencent.supersonic.common.pojo.ModelRela;
+import com.tencent.supersonic.semantic.api.model.pojo.Identify;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import java.util.List;
+
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
@@ -13,5 +15,20 @@ public class ModelSchemaResp extends ModelResp {
 
     private List<MetricSchemaResp> metrics;
     private List<DimSchemaResp> dimensions;
+    private List<ModelRela> modelRelas;
+
+    public DimSchemaResp getPrimaryKey() {
+        Identify identify = getPrimaryIdentify();
+        if (identify == null) {
+            return null;
+        }
+        for (DimSchemaResp dimension : dimensions) {
+            if (identify.getBizName().equals(dimension.getBizName())) {
+                dimension.setEntityAlias(identify.getEntityNames());
+                return dimension;
+            }
+        }
+        return null;
+    }
 
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DataModelYamlTpl.java

@@ -5,7 +5,7 @@ import lombok.Data;
 
 
 @Data
-public class DatasourceYamlTpl {
+public class DataModelYamlTpl {
 
     private String name;
 

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2SQLReq.java

@@ -1,17 +1,30 @@
 package com.tencent.supersonic.semantic.api.query.request;
 
-import java.util.Map;
+import com.google.common.collect.Lists;
 import lombok.Data;
 import lombok.ToString;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 @Data
 @ToString
 public class QueryS2SQLReq {
 
-    private Long modelId;
+    private Set<Long> modelIds;
 
     private String sql;
 
     private Map<String, String> variables;
 
+    public void setModelId(Long modelId) {
+        modelIds = new HashSet<>();
+        modelIds.add(modelId);
+    }
+
+    public List<Long> getModelIds() {
+        return Lists.newArrayList(modelIds);
+    }
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java

@@ -14,11 +14,6 @@ import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
 import com.tencent.supersonic.semantic.api.query.pojo.Cache;
 import com.tencent.supersonic.semantic.api.query.pojo.Param;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
@@ -42,12 +37,20 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 
 @Data
 @Slf4j
 public class QueryStructReq {
 
-    private Long modelId;
+    private Set<Long> modelIds;
 
     private String modelName;
     private List<String> groups = new ArrayList<>();
@@ -70,6 +73,19 @@ public class QueryStructReq {
      */
     private String correctS2SQL;
 
+    public void setModelId(Long modelId) {
+        modelIds = new HashSet<>();
+        modelIds.add(modelId);
+    }
+
+    public List<Long> getModelIds() {
+        return Lists.newArrayList(modelIds);
+    }
+
+    public Set<Long> getModelIdSet() {
+        return modelIds;
+    }
+
     public List<String> getGroups() {
         if (!CollectionUtils.isEmpty(this.groups)) {
             this.groups = groups.stream().filter(group -> !Strings.isEmpty(group)).collect(Collectors.toList());
@@ -107,7 +123,7 @@ public class QueryStructReq {
     public String toCustomizedString() {
         StringBuilder stringBuilder = new StringBuilder("{");
         stringBuilder.append("\"modelId\":")
-                .append(modelId);
+                .append(modelIds);
         stringBuilder.append(",\"groups\":")
                 .append(groups);
         stringBuilder.append(",\"aggregators\":")
@@ -139,7 +155,7 @@ public class QueryStructReq {
     public String toString() {
         final StringBuilder sb = new StringBuilder("{");
         sb.append("\"modelId\":")
-                .append(modelId);
+                .append(modelIds);
         sb.append(",\"groups\":")
                 .append(groups);
         sb.append(",\"aggregators\":")
@@ -179,7 +195,7 @@ public class QueryStructReq {
 
         QueryS2SQLReq result = new QueryS2SQLReq();
         result.setSql(sql);
-        result.setModelId(queryStructReq.getModelId());
+        result.setModelIds(queryStructReq.getModelIdSet());
         result.setVariables(new HashMap<>());
         return result;
     }