topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless)(Chat) Change View to DataSet (#782)

Browse Source 
* (improvement)(Headless)(Chat) Change view to dataSet



---------

Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
LXW
2024-03-04 11:48:41 +08:00
committed by GitHub
parent b29e429271
commit a41da3f5fe
184 changed files with 1628 additions and 1532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java
View File

@@ -4,8 +4,8 @@ import lombok.Data;
import java.util.List;
@Data
public class ViewDetail {
public class DataSetDetail {
private List<ViewModelConfig> viewModelConfigs;
private List<DataSetModelConfig> dataSetModelConfigs;
}

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java
View File

@@ -10,7 +10,7 @@ import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class ViewModelConfig {
public class DataSetModelConfig {
private Long id;
@@ -20,7 +20,7 @@ public class ViewModelConfig {
private List<Long> dimensions = Lists.newArrayList();
public ViewModelConfig(Long id, List<Long> dimensions, List<Long> metrics) {
public DataSetModelConfig(Long id, List<Long> dimensions, List<Long> metrics) {
this.id = id;
this.metrics = metrics;
this.dimensions = dimensions;

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java
View File

@@ -25,8 +25,8 @@ public class QueryParam {
private QueryType queryType;
private String s2SQL;
private String correctS2SQL;
private Long viewId;
private String viewName;
private Long dataSetId;
private String dataSetName;
private Set<Long> modelIds = new HashSet<>();
private List<Param> params = new ArrayList<>();

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java
View File

@@ -16,7 +16,7 @@ public class QueryStat {
private String traceId;
private Long modelId;
private Long viewId;
private Long dataSetId;
private String user;
private String createdAt;
/**
@@ -93,8 +93,8 @@ public class QueryStat {
return this;
}
public QueryStat setViewId(Long viewId) {
this.viewId = viewId;
public QueryStat setDataSetId(Long dataSetId) {
this.dataSetId = dataSetId;
return this;
}

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java
View File

@@ -17,7 +17,7 @@ import java.util.List;
@NoArgsConstructor
public class SchemaElement implements Serializable {
private Long view;
private Long dataSet;
private Long model;
private Long id;
private String name;
@@ -41,7 +41,7 @@ public class SchemaElement implements Serializable {
return false;
}
SchemaElement schemaElement = (SchemaElement) o;
return Objects.equal(view, schemaElement.view) && Objects.equal(id,
return Objects.equal(dataSet, schemaElement.dataSet) && Objects.equal(id,
schemaElement.id) && Objects.equal(name, schemaElement.name)
&& Objects.equal(bizName, schemaElement.bizName)
&& Objects.equal(type, schemaElement.type);
@@ -49,7 +49,7 @@ public class SchemaElement implements Serializable {
@Override
public int hashCode() {
return Objects.hashCode(view, id, name, bizName, type);
return Objects.hashCode(dataSet, id, name, bizName, type);
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.api.pojo;
public enum SchemaElementType {
VIEW,
DATASET,
METRIC,
DIMENSION,
VALUE,

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.enums;
/**
* model datasource define type:
* sql_query : view sql begin as select
* sql_query : dataSet sql begin as select
* table_query: dbName.tableName
*/
public enum ModelDefineType {

33
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java Normal file
View File

@@ -0,0 +1,33 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Data
@NoArgsConstructor
public class DataSetFilterReq {
protected boolean defaultAll = true;
protected List<Long> dataSetIds = Lists.newArrayList();
public DataSetFilterReq(Long dataSetId) {
addDataSet(dataSetId);
}
public List<Long> getDataSetIds() {
if (CollectionUtils.isEmpty(dataSetIds) && !defaultAll) {
return Lists.newArrayList(-1L);
}
return dataSetIds;
}
public void addDataSet(Long dataSetId) {
dataSetIds.add(dataSetId);
}
}

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java
View File

@@ -2,17 +2,17 @@ package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import lombok.Data;
import java.util.List;
@Data
public class ViewReq extends SchemaItem {
public class DataSetReq extends SchemaItem {
private Long domainId;
private ViewDetail viewDetail;
private DataSetDetail dataSetDetail;
private String alias;

11
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryViewReq.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java
View File

@@ -7,17 +7,18 @@ import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Param;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import java.util.ArrayList;
import java.util.List;
@Data
@ToString
public class QueryViewReq {
public class QueryDataSetReq {
private Long viewId;
private String viewName;
private Long dataSetId;
private String dataSetName;
private String sql;
private boolean needAuth = true;
private List<Param> params = new ArrayList<>();

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java
View File

@@ -27,7 +27,7 @@ public class QueryMultiStructReq extends SemanticQueryReq {
if (CollectionUtils.isEmpty(this.getQueryStructReqs())) {
return null;
}
return this.getQueryStructReqs().get(0).getViewId();
return this.getQueryStructReqs().get(0).getDataSetId();
}
public Cache getCacheInfo() {

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java
View File

@@ -12,8 +12,8 @@ public class QuerySqlReq extends SemanticQueryReq {
@Override
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"viewId\":")
.append(viewId);
stringBuilder.append("\"dataSetId\":")
.append(dataSetId);
stringBuilder.append("\"modelIds\":")
.append(modelIds);
stringBuilder.append(",\"params\":")

18
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -94,8 +94,8 @@ public class QueryStructReq extends SemanticQueryReq {
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"viewId\":")
.append(viewId);
stringBuilder.append("\"dataSetId\":")
.append(dataSetId);
stringBuilder.append("\"modelIds\":")
.append(modelIds);
stringBuilder.append(",\"groups\":")
@@ -127,8 +127,8 @@ public class QueryStructReq extends SemanticQueryReq {
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
sb.append("\"viewId\":")
.append(viewId);
sb.append("\"dataSetId\":")
.append(dataSetId);
sb.append("\"modelIds\":")
.append(modelIds);
sb.append(",\"groups\":")
@@ -172,7 +172,7 @@ public class QueryStructReq extends SemanticQueryReq {
QuerySqlReq result = new QuerySqlReq();
result.setSql(sql);
result.setViewId(this.getViewId());
result.setDataSetId(this.getDataSetId());
result.setModelIds(this.getModelIdSet());
result.setParams(new ArrayList<>());
return result;
@@ -276,11 +276,11 @@ public class QueryStructReq extends SemanticQueryReq {
}
public String getTableName() {
if (StringUtils.isNotBlank(viewName)) {
return viewName;
if (StringUtils.isNotBlank(dataSetName)) {
return dataSetName;
}
if (viewId != null) {
return Constants.TABLE_PREFIX + viewId;
if (dataSetId != null) {
return Constants.TABLE_PREFIX + dataSetId;
}
return Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_");
}

11
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java
View File

@@ -5,14 +5,15 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Data
@Slf4j
@ToString
@@ -32,8 +33,8 @@ public class QueryTagReq extends SemanticQueryReq {
@Override
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"viewId\":")
.append(viewId);
stringBuilder.append("\"dataSetId\":")
.append(dataSetId);
stringBuilder.append("\"modelIds\":")
.append(modelIds);
stringBuilder.append(",\"groups\":")

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java
View File

@@ -8,7 +8,7 @@ import java.util.List;
@Data
public class SchemaFilterReq {
private Long viewId;
private Long dataSetId;
private List<Long> modelIds = Lists.newArrayList();

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
View File

@@ -19,9 +19,9 @@ public abstract class SemanticQueryReq {
protected boolean needAuth = true;
protected Long viewId;
protected Long dataSetId;
protected String viewName;
protected String dataSetName;
protected Set<Long> modelIds = new HashSet<>();

33
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java
View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Data
@NoArgsConstructor
public class ViewFilterReq {
protected boolean defaultAll = true;
protected List<Long> viewIds = Lists.newArrayList();
public ViewFilterReq(Long viewId) {
addView(viewId);
}
public List<Long> getViewIds() {
if (CollectionUtils.isEmpty(viewIds) && !defaultAll) {
return Lists.newArrayList(-1L);
}
return viewIds;
}
public void addView(Long viewId) {
viewIds.add(viewId);
}
}

24
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.ViewModelConfig;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
import lombok.Data;
import org.springframework.util.CollectionUtils;
@@ -14,11 +14,11 @@ import java.util.List;
import java.util.stream.Collectors;
@Data
public class ViewResp extends SchemaItem {
public class DataSetResp extends SchemaItem {
private Long domainId;
private ViewDetail viewDetail;
private DataSetDetail dataSetDetail;
private String alias;
@@ -31,31 +31,31 @@ public class ViewResp extends SchemaItem {
private QueryConfig queryConfig;
public List<Long> getAllMetrics() {
return getViewModelConfigs().stream().map(ViewModelConfig::getMetrics)
return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics)
.flatMap(Collection::stream).collect(Collectors.toList());
}
public List<Long> getAllDimensions() {
return getViewModelConfigs().stream().map(ViewModelConfig::getDimensions)
return getDataSetModelConfigs().stream().map(DataSetModelConfig::getDimensions)
.flatMap(Collection::stream).collect(Collectors.toList());
}
public List<Long> getAllModels() {
return getViewModelConfigs().stream().map(ViewModelConfig::getId)
return getDataSetModelConfigs().stream().map(DataSetModelConfig::getId)
.collect(Collectors.toList());
}
public List<Long> getAllIncludeAllModels() {
return getViewModelConfigs().stream().filter(ViewModelConfig::isIncludesAll)
.map(ViewModelConfig::getId)
return getDataSetModelConfigs().stream().filter(DataSetModelConfig::isIncludesAll)
.map(DataSetModelConfig::getId)
.collect(Collectors.toList());
}
private List<ViewModelConfig> getViewModelConfigs() {
if (viewDetail == null || CollectionUtils.isEmpty(viewDetail.getViewModelConfigs())) {
private List<DataSetModelConfig> getDataSetModelConfigs() {
if (dataSetDetail == null || CollectionUtils.isEmpty(dataSetDetail.getDataSetModelConfigs())) {
return Lists.newArrayList();
}
return viewDetail.getViewModelConfigs();
return dataSetDetail.getDataSetModelConfigs();
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java
View File

@@ -11,7 +11,7 @@ import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class ViewSchemaResp extends ViewResp {
public class DataSetSchemaResp extends DataSetResp {
private List<MetricSchemaResp> metrics = Lists.newArrayList();
private List<DimSchemaResp> dimensions = Lists.newArrayList();

8
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java
View File

@@ -17,7 +17,7 @@ import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
@NoArgsConstructor
public class SemanticSchemaResp {
private Long viewId;
private Long dataSetId;
private List<Long> modelIds;
private SchemaType schemaType;
private List<MetricSchemaResp> metrics = Lists.newArrayList();
@@ -25,14 +25,14 @@ public class SemanticSchemaResp {
private List<TagResp> tags = Lists.newArrayList();
private List<ModelRela> modelRelas = Lists.newArrayList();
private List<ModelResp> modelResps = Lists.newArrayList();
private ViewResp viewResp;
private DataSetResp dataSetResp;
private DatabaseResp databaseResp;
public String getSchemaKey() {
if (viewId == null) {
if (dataSetId == null) {
return String.format("%s_%s", schemaType, StringUtils.join(modelIds, UNDERLINE));
}
return String.format("%s_%s", schemaType, viewId);
return String.format("%s_%s", schemaType, dataSetId);
}

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DataSetInfoStat.java Normal file
View File

@@ -0,0 +1,22 @@
package com.tencent.supersonic.headless.core.knowledge;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.io.Serializable;
@Data
@ToString
@Builder
public class DataSetInfoStat implements Serializable {
private long dataSetCount;
private long metricDataSetCount;
private long dimensionDataSetCount;
private long dimensionValueDataSetCount;
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java
View File

@@ -58,7 +58,7 @@ public class SearchService {
.collect(Collectors.toList());
for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
List<String> natures = hanlpMapResult.getNatures().stream()
.map(nature -> NatureHelper.changeModel2View(nature, modelIdToViewIds))
.map(nature -> NatureHelper.changeModel2DataSet(nature, modelIdToViewIds))
.flatMap(Collection::stream).collect(Collectors.toList());
hanlpMapResult.setNatures(natures);
}

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java
View File

@@ -1,22 +0,0 @@
package com.tencent.supersonic.headless.core.knowledge;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.io.Serializable;
@Data
@ToString
@Builder
public class ViewInfoStat implements Serializable {
private long viewCount;
private long metricViewCount;
private long dimensionViewCount;
private long dimensionValueViewCount;
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java
View File

@@ -21,13 +21,13 @@ public class ModelWordBuilder extends BaseWordBuilder {
public List<DictWord> doGet(String word, SchemaElement schemaElement) {
List<DictWord> result = Lists.newArrayList();
//modelName
DictWord dictWord = buildDictWord(word, schemaElement.getView());
DictWord dictWord = buildDictWord(word, schemaElement.getDataSet());
result.add(dictWord);
//alias
List<String> aliasList = schemaElement.getAlias();
if (CollectionUtils.isNotEmpty(aliasList)) {
for (String alias : aliasList) {
result.add(buildDictWord(alias, schemaElement.getView()));
result.add(buildDictWord(alias, schemaElement.getDataSet()));
}
}
return result;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java
View File

@@ -224,7 +224,7 @@ public class HanlpHelper {
public static List<S2Term> transform2ApiTerm(Term term, Map<Long, List<Long>> modelIdToViewIds) {
List<S2Term> s2Terms = Lists.newArrayList();
List<String> natures = NatureHelper.changeModel2View(String.valueOf(term.getNature()), modelIdToViewIds);
List<String> natures = NatureHelper.changeModel2DataSet(String.valueOf(term.getNature()), modelIdToViewIds);
for (String nature : natures) {
S2Term s2Term = new S2Term();
BeanUtils.copyProperties(term, s2Term);

56
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java
View File

@@ -5,7 +5,7 @@ import com.hankcs.hanlp.corpus.tag.Nature;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat;
import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -41,7 +41,7 @@ public class NatureHelper {
result = SchemaElementType.ENTITY;
break;
case VIEW:
result = SchemaElementType.VIEW;
result = SchemaElementType.DATASET;
break;
case VALUE:
result = SchemaElementType.VALUE;
@@ -55,12 +55,12 @@ public class NatureHelper {
return result;
}
private static boolean isViewOrEntity(S2Term term, Integer model) {
private static boolean isDataSetOrEntity(S2Term term, Integer model) {
return (DictWordType.NATURE_SPILT + model).equals(term.nature.toString()) || term.nature.toString()
.endsWith(DictWordType.ENTITY.getType());
}
public static Integer getViewByNature(Nature nature) {
public static Integer getDataSetByNature(Nature nature) {
if (nature.startsWith(DictWordType.NATURE_SPILT)) {
String[] dimensionValues = nature.toString().split(DictWordType.NATURE_SPILT);
if (StringUtils.isNumeric(dimensionValues[1])) {
@@ -70,7 +70,7 @@ public class NatureHelper {
return 0;
}
public static Long getViewId(String nature) {
public static Long getDataSetId(String nature) {
try {
String[] split = nature.split(DictWordType.NATURE_SPILT);
if (split.length <= 1) {
@@ -96,13 +96,13 @@ public class NatureHelper {
return null;
}
private static Nature changeModel2View(String nature, Long viewId) {
private static Nature changeModel2DataSet(String nature, Long dataSetId) {
try {
String[] split = nature.split(DictWordType.NATURE_SPILT);
if (split.length <= 1) {
return null;
}
split[1] = String.valueOf(viewId);
split[1] = String.valueOf(dataSetId);
return Nature.create(StringUtils.join(split, DictWordType.NATURE_SPILT));
} catch (NumberFormatException e) {
log.error("", e);
@@ -110,17 +110,17 @@ public class NatureHelper {
return null;
}
public static List<String> changeModel2View(String nature, Map<Long, List<Long>> modelIdToViewIds) {
public static List<String> changeModel2DataSet(String nature, Map<Long, List<Long>> modelIdToDataSetIds) {
Long modelId = getModelId(nature);
List<Long> viewIds = modelIdToViewIds.get(modelId);
if (CollectionUtils.isEmpty(viewIds)) {
List<Long> dataSetIds = modelIdToDataSetIds.get(modelId);
if (CollectionUtils.isEmpty(dataSetIds)) {
return Lists.newArrayList();
}
return viewIds.stream().map(viewId -> String.valueOf(changeModel2View(nature, viewId)))
return dataSetIds.stream().map(dataSetId -> String.valueOf(changeModel2DataSet(nature, dataSetId)))
.collect(Collectors.toList());
}
public static boolean isDimensionValueViewId(String nature) {
public static boolean isDimensionValueDataSetId(String nature) {
if (StringUtils.isEmpty(nature)) {
return false;
}
@@ -135,21 +135,21 @@ public class NatureHelper {
&& StringUtils.isNumeric(split[1]);
}
public static ViewInfoStat getViewStat(List<S2Term> terms) {
return ViewInfoStat.builder()
.viewCount(getViewCount(terms))
.dimensionViewCount(getDimensionCount(terms))
.metricViewCount(getMetricCount(terms))
.dimensionValueViewCount(getDimensionValueCount(terms))
public static DataSetInfoStat getDataSetStat(List<S2Term> terms) {
return DataSetInfoStat.builder()
.dataSetCount(getDataSetCount(terms))
.dimensionDataSetCount(getDimensionCount(terms))
.metricDataSetCount(getMetricCount(terms))
.dimensionValueDataSetCount(getDimensionValueCount(terms))
.build();
}
private static long getViewCount(List<S2Term> terms) {
return terms.stream().filter(term -> isViewOrEntity(term, getViewByNature(term.nature))).count();
private static long getDataSetCount(List<S2Term> terms) {
return terms.stream().filter(term -> isDataSetOrEntity(term, getDataSetByNature(term.nature))).count();
}
private static long getDimensionValueCount(List<S2Term> terms) {
return terms.stream().filter(term -> isDimensionValueViewId(term.nature.toString())).count();
return terms.stream().filter(term -> isDimensionValueDataSetId(term.nature.toString())).count();
}
private static long getDimensionCount(List<S2Term> terms) {
@@ -169,13 +169,13 @@ public class NatureHelper {
* @param terms
* @return
*/
public static Map<Long, Map<DictWordType, Integer>> getViewToNatureStat(List<S2Term> terms) {
public static Map<Long, Map<DictWordType, Integer>> getDataSetToNatureStat(List<S2Term> terms) {
Map<Long, Map<DictWordType, Integer>> modelToNature = new HashMap<>();
terms.stream().filter(
term -> term.nature.startsWith(DictWordType.NATURE_SPILT)
).forEach(term -> {
DictWordType dictWordType = DictWordType.getNatureType(String.valueOf(term.nature));
Long model = getViewId(String.valueOf(term.nature));
Long model = getDataSetId(String.valueOf(term.nature));
Map<DictWordType, Integer> natureTypeMap = new HashMap<>();
natureTypeMap.put(dictWordType, 1);
@@ -196,15 +196,15 @@ public class NatureHelper {
return modelToNature;
}
public static List<Long> selectPossibleViews(List<S2Term> terms) {
Map<Long, Map<DictWordType, Integer>> modelToNatureStat = getViewToNatureStat(terms);
Integer maxViewTypeSize = modelToNatureStat.entrySet().stream()
public static List<Long> selectPossibleDataSets(List<S2Term> terms) {
Map<Long, Map<DictWordType, Integer>> modelToNatureStat = getDataSetToNatureStat(terms);
Integer maxDataSetTypeSize = modelToNatureStat.entrySet().stream()
.max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size())
.orElse(null);
if (Objects.isNull(maxViewTypeSize) || maxViewTypeSize == 0) {
if (Objects.isNull(maxDataSetTypeSize) || maxDataSetTypeSize == 0) {
return new ArrayList<>();
}
return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxViewTypeSize)
return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxDataSetTypeSize)
.map(entry -> entry.getKey()).collect(Collectors.toList());
}

59
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java
View File

@@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component
@Slf4j
@Primary
@@ -28,8 +29,8 @@ public class DefaultQueryParser implements QueryParser {
public void parse(QueryStatement queryStatement) throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getViewQueryParam())) {
queryStatement.setViewQueryParam(new ViewQueryParam());
if (Objects.isNull(queryStatement.getDataSetQueryParam())) {
queryStatement.setDataSetQueryParam(new DataSetQueryParam());
}
if (Objects.isNull(queryStatement.getMetricQueryParam())) {
queryStatement.setMetricQueryParam(new MetricQueryParam());
@@ -41,10 +42,10 @@ public class DefaultQueryParser implements QueryParser {
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getViewQueryParam(),
log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getDataSetQueryParam(),
queryStatement.getMetricQueryParam());
if (!queryStatement.getViewQueryParam().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getViewQueryParam(), queryStatement);
if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getDataSetQueryParam(), queryStatement);
} else {
queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
@@ -61,37 +62,37 @@ public class DefaultQueryParser implements QueryParser {
queryStatement.setSql(querySql);
}
public QueryStatement parser(ViewQueryParam viewQueryParam, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", viewQueryParam);
public QueryStatement parser(DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", dataSetQueryParam);
try {
if (!CollectionUtils.isEmpty(viewQueryParam.getTables())) {
if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = viewQueryParam.getTables().size() == 1;
for (MetricTable metricTable : viewQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable, viewQueryParam, queryStatement);
if (isSingleTable && Objects.nonNull(tableSql.getViewSimplifySql())
&& !tableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(tableSql.getViewSimplifySql());
queryStatement.setViewQueryParam(viewQueryParam);
Boolean isSingleTable = dataSetQueryParam.getTables().size() == 1;
for (MetricTable metricTable : dataSetQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable, dataSetQueryParam, queryStatement);
if (isSingleTable && Objects.nonNull(tableSql.getDataSetQueryParam())
&& !tableSql.getDataSetSimplifySql().isEmpty()) {
queryStatement.setSql(tableSql.getDataSetSimplifySql());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (viewQueryParam.isSupportWith()) {
if (dataSetQueryParam.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + viewQueryParam.getSql();
Collectors.toList())) + "\n" + dataSetQueryParam.getSql();
} else {
sql = viewQueryParam.getSql();
sql = dataSetQueryParam.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (viewQueryParam.isWithAlias() ? "" : tb[0]), -1);
"(" + tb[1] + ") " + (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setViewQueryParam(viewQueryParam);
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
}
@@ -119,7 +120,7 @@ public class DefaultQueryParser implements QueryParser {
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
ViewQueryParam viewQueryParam,
DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) throws Exception {
MetricQueryParam metricReq = new MetricQueryParam();
metricReq.setMetrics(metricTable.getMetrics());
@@ -131,11 +132,11 @@ public class DefaultQueryParser implements QueryParser {
tableSql.setMetricQueryParam(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setViewId(queryStatement.getViewId());
tableSql.setDataSetId(queryStatement.getDataSetId());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(viewQueryParam.getSql());
tableSql.setViewAlias(metricTable.getAlias());
tableSql.setDataSetSql(dataSetQueryParam.getSql());
tableSql.setDataSetAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -37,14 +37,14 @@ public class CalciteSqlParser implements SqlParser {
queryStatement.setSql(aggBuilder.getSql(engineType));
queryStatement.setSourceId(aggBuilder.getSourceId());
if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize()
&& Objects.nonNull(queryStatement.getViewAlias()) && !queryStatement.getViewAlias().isEmpty()) {
&& Objects.nonNull(queryStatement.getDataSetAlias()) && !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql
String simplifySql = aggBuilder.simplify(
getSqlByView(aggBuilder.getSql(engineType), queryStatement.getViewSql(),
queryStatement.getViewAlias()), engineType);
getSqlByDataSet(aggBuilder.getSql(engineType), queryStatement.getDataSetSql(),
queryStatement.getDataSetAlias()), engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
log.debug("simplifySql [{}]", simplifySql);
queryStatement.setViewSimplifySql(simplifySql);
queryStatement.setDataSetSimplifySql(simplifySql);
}
}
return queryStatement;
@@ -62,7 +62,7 @@ public class CalciteSqlParser implements SqlParser {
return semanticSchema;
}
private String getSqlByView(String sql, String viewSql, String viewAlias) {
return String.format("with %s as (%s) %s", viewAlias, sql, viewSql);
private String getSqlByDataSet(String sql, String dataSetSql, String dataSetAlias) {
return String.format("with %s as (%s) %s", dataSetAlias, sql, dataSetSql);
}
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java
View File

@@ -53,16 +53,16 @@ public class SchemaBuilder {
.withRowCount(1)
.build();
schema.add(MATERIALIZATION_SYS_SOURCE, srcTable);
DataSourceTable viewTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW)
DataSourceTable dataSetTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW)
.addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE)
.addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT)
.withRowCount(1)
.build();
schema.add(MATERIALIZATION_SYS_VIEW, viewTable);
schema.add(MATERIALIZATION_SYS_VIEW, dataSetTable);
return rootSchema;
}
public static void addSourceView(CalciteSchema viewSchema, String dbSrc, String tbSrc, Set<String> dates,
public static void addSourceView(CalciteSchema dataSetSchema, String dbSrc, String tbSrc, Set<String> dates,
Set<String> dimensions, Set<String> metrics) {
String tb = tbSrc.toLowerCase();
String db = dbSrc.toLowerCase();
@@ -80,14 +80,14 @@ public class SchemaBuilder {
.withRowCount(1)
.build();
if (Objects.nonNull(db) && !db.isEmpty()) {
SchemaPlus schemaPlus = viewSchema.plus().getSubSchema(db);
SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db);
if (Objects.isNull(schemaPlus)) {
viewSchema.plus().add(db, new AbstractSchema());
schemaPlus = viewSchema.plus().getSubSchema(db);
dataSetSchema.plus().add(db, new AbstractSchema());
schemaPlus = dataSetSchema.plus().getSubSchema(db);
}
schemaPlus.add(tb, srcTable);
} else {
viewSchema.add(tb, srcTable);
dataSetSchema.add(tb, srcTable);
}
}
}

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/ViewExpanderImpl.java
View File

@@ -1,10 +1,11 @@
package com.tencent.supersonic.headless.core.parser.calcite.schema;
import java.util.List;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.type.RelDataType;
import java.util.List;
/**
* customize the ViewExpander
*/
@@ -14,7 +15,7 @@ public class ViewExpanderImpl implements RelOptTable.ViewExpander {
@Override
public RelRoot expandView(RelDataType rowType, String queryString, List<String> schemaPath,
List<String> viewPath) {
List<String> dataSetPath) {
return null;
}
}

27
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
@@ -13,6 +12,18 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -24,16 +35,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DataSourceNode extends SemanticNode {
@@ -126,9 +127,9 @@ public class DataSourceNode extends SemanticNode {
return build(datasource, scope);
}
EngineType engineType = EngineType.fromString(datasource.getType());
SqlNode view = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope),
SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope),
new SqlNodeList(getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)), SqlParserPos.ZERO);
return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, view);
return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, dataSet);
}
public static List<SqlNode> getExtendField(Set<String> exprList, SqlValidatorScope scope, EngineType engineType)

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java
View File

@@ -12,11 +12,11 @@ import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlWriter;
/**
* extend node to handle lateral explode view
* extend node to handle lateral explode dataSet
*/
public class LateralViewExplodeNode extends ExtendNode {
public final String sqlNameView = "view";
public final String sqlNameView = "dataSet";
public final String sqlNameExplode = "explode";
public LateralViewExplodeNode() {

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java
View File

@@ -34,7 +34,7 @@ import java.util.UUID;
import java.util.stream.Collectors;
/**
* process the table view from the defined data model schema
* process the table dataSet from the defined data model schema
*/
@Slf4j
public class SourceRender extends Renderer {

23
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -42,14 +42,15 @@ public class CalculateAggConverter implements HeadlessConverter {
String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
}
public ViewQueryParam generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement,
EngineType engineTypeEnum, String version)
throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
// 同环比
if (isRatioAccept(queryParam)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
}
ViewQueryParam sqlCommand = new ViewQueryParam();
DataSetQueryParam sqlCommand = new DataSetQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
@@ -105,13 +106,13 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
ViewQueryParam sqlCommend = queryStatement.getViewQueryParam();
DataSetQueryParam sqlCommend = queryStatement.getDataSetQueryParam();
Database database = queryStatement.getSemanticModel().getDatabase();
ViewQueryParam viewQueryParam = generateSqlCommend(queryStatement,
DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
sqlCommend.setSql(viewQueryParam.getSql());
sqlCommend.setTables(viewQueryParam.getTables());
sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
sqlCommend.setSql(dataSetQueryParam.getSql());
sqlCommend.setTables(dataSetQueryParam.getTables());
sqlCommend.setSupportWith(dataSetQueryParam.isSupportWith());
}
/**
@@ -128,13 +129,13 @@ public class CalculateAggConverter implements HeadlessConverter {
return false;
}
public ViewQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version)
public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version)
throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
check(queryParam);
queryStatement.setEnableOptimize(false);
ViewQueryParam sqlCommand = new ViewQueryParam();
DataSetQueryParam sqlCommand = new DataSetQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java
View File

@@ -27,8 +27,8 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
return !Objects.isNull(queryStatement.getViewQueryParam())
&& !StringUtils.isBlank(queryStatement.getViewQueryParam().getSql());
return !Objects.isNull(queryStatement.getDataSetQueryParam())
&& !StringUtils.isBlank(queryStatement.getDataSetQueryParam().getSql());
}
@Override
@@ -39,14 +39,14 @@ public class DefaultDimValueConverter implements HeadlessConverter {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
String sql = queryStatement.getViewQueryParam().getSql();
String sql = queryStatement.getDataSetQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql)
.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;
}
MetricTable metricTable = queryStatement.getViewQueryParam()
MetricTable metricTable = queryStatement.getDataSetQueryParam()
.getTables().stream().findFirst().orElse(null);
List<Expression> expressions = Lists.newArrayList();
for (Dimension dimension : dimensions) {
@@ -63,7 +63,7 @@ public class DefaultDimValueConverter implements HeadlessConverter {
}
}
sql = SqlAddHelper.addWhere(sql, expressions);
queryStatement.getViewQueryParam().setSql(sql);
queryStatement.getDataSetQueryParam().setSql(sql);
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java
View File

@@ -6,7 +6,7 @@ import lombok.Data;
import java.util.List;
@Data
public class ViewQueryParam {
public class DataSetQueryParam {
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java
View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -11,7 +11,7 @@ import java.util.List;
@Data
public class QueryStatement {
private Long viewId;
private Long dataSetId;
private List<Long> modelIds;
private String sql = "";
private String sourceId = "";
@@ -19,15 +19,15 @@ public class QueryStatement {
private Boolean ok;
private QueryParam queryParam;
private MetricQueryParam metricQueryParam;
private ViewQueryParam viewQueryParam;
private DataSetQueryParam dataSetQueryParam;
private Integer status = 0;
private Boolean isS2SQL = false;
private List<ImmutablePair<String, String>> timeRanges;
private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime;
private String viewSql = "";
private String viewAlias = "";
private String viewSimplifySql = "";
private String dataSetSql = "";
private String dataSetAlias = "";
private String dataSetSimplifySql = "";
private Boolean enableLimitWrapper = false;
private SemanticModel semanticModel;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
View File

@@ -37,10 +37,10 @@ public class MetricDrillDownChecker {
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] objects = joinPoint.getArgs();
QueryStatement queryStatement = (QueryStatement) objects[0];
if (queryStatement.getViewQueryParam() == null) {
if (queryStatement.getDataSetQueryParam() == null) {
return joinPoint.proceed();
}
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getViewQueryParam().getSql());
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getDataSetQueryParam().getSql());
return joinPoint.proceed();
}

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -30,13 +30,13 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -84,7 +84,7 @@ public class S2DataPermissionAspect {
@Autowired
private SchemaService schemaService;
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@Autowired
private AuthService authService;
@@ -104,7 +104,7 @@ public class S2DataPermissionAspect {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = getModelsInView(queryReq);
List<Long> modelIds = getModelsInDataSet(queryReq);
// determine whether admin of the model
if (doModelAdmin(user, modelIds)) {
@@ -129,21 +129,21 @@ public class S2DataPermissionAspect {
// fetch data permission meta information
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(querySqlReq.getModelIds());
filter.setViewId(querySqlReq.getViewId());
filter.setDataSetId(querySqlReq.getDataSetId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInView = semanticSchemaResp.getModelResps().stream()
List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp);
log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInView, sensitiveResReq);
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInDataSet, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView);
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
@@ -157,7 +157,7 @@ public class S2DataPermissionAspect {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
}
// if the column has no permission, hit *
@@ -166,7 +166,7 @@ public class S2DataPermissionAspect {
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply);
addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
@@ -228,23 +228,23 @@ public class S2DataPermissionAspect {
// fetch data permission meta information
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(queryStructReq.getModelIds());
filter.setViewId(queryStructReq.getViewId());
filter.setDataSetId(queryStructReq.getDataSetId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInView = semanticSchemaResp.getModelResps().stream()
List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInView, sensitiveResReq);
log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInDataSet, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIdInView, sensitiveResReq);
modelIdInDataSet, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView);
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
@@ -258,7 +258,7 @@ public class S2DataPermissionAspect {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
}
// if the column has no permission, hit *
@@ -266,7 +266,7 @@ public class S2DataPermissionAspect {
.collect(Collectors.toSet());
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply);
addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
@@ -557,11 +557,11 @@ public class S2DataPermissionAspect {
}
}
private List<Long> getModelsInView(SemanticQueryReq queryReq) {
private List<Long> getModelsInDataSet(SemanticQueryReq queryReq) {
List<Long> modelIds = queryReq.getModelIds();
if (queryReq.getViewId() != null) {
ViewResp viewResp = viewService.getView(queryReq.getViewId());
modelIds = viewResp.getAllModels();
if (queryReq.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(queryReq.getDataSetId());
modelIds = dataSetResp.getAllModels();
}
return modelIds;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java
View File

@@ -7,8 +7,8 @@ import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_view")
public class ViewDO {
@TableName("s2_data_set")
public class DataSetDO {
@TableId(type = IdType.AUTO)
private Long id;
@@ -25,7 +25,7 @@ public class ViewDO {
private String alias;
private String viewDetail;
private String dataSetDetail;
private Date createdAt;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ViewDOMapper extends BaseMapper<ViewDO> {
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java
View File

@@ -48,9 +48,9 @@ public class StatRepositoryImpl implements StatRepository {
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
if (Objects.nonNull(stat.getViewId())) {
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getViewId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getViewId());
if (Objects.nonNull(stat.getDataSetId())) {
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getDataSetId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getDataSetId());
}
});
map.forEach((k, v) -> {
@@ -69,13 +69,13 @@ public class StatRepositoryImpl implements StatRepository {
return statMapper.getStatInfo(itemUseCommend);
}
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long viewId) {
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long dataSetId) {
if (Strings.isNotEmpty(dimensions)) {
try {
List<String> dimensionList = mapper.readValue(dimensions, new TypeReference<List<String>>() {
});
dimensionList.stream().forEach(dimension -> {
String key = viewId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
View File

@@ -22,7 +22,7 @@ public class MetaFilter {
private Long domainId;
private Long viewId;
private Long dataSetId;
private Integer sensitiveLevel;
@@ -51,7 +51,7 @@ public class MetaFilter {
&& Objects.equal(bizName, that.bizName) && Objects.equal(
createdBy, that.createdBy) && Objects.equal(modelIds, that.modelIds)
&& Objects.equal(domainId, that.domainId) && Objects.equal(
viewId, that.viewId) && Objects.equal(sensitiveLevel, that.sensitiveLevel)
dataSetId, that.dataSetId) && Objects.equal(sensitiveLevel, that.sensitiveLevel)
&& Objects.equal(status, that.status) && Objects.equal(key,
that.key) && Objects.equal(ids, that.ids) && Objects.equal(
fieldsDepend, that.fieldsDepend);
@@ -59,7 +59,7 @@ public class MetaFilter {
@Override
public int hashCode() {
return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId, viewId, sensitiveLevel, status, key,
ids, fieldsDepend);
return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId,
dataSetId, sensitiveLevel, status, key, ids, fieldsDepend);
}
}

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java
View File

@@ -2,10 +2,10 @@ package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -21,43 +21,43 @@ import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/view")
public class ViewController {
@RequestMapping("/api/semantic/dataSet")
public class DataSetController {
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@PostMapping
public ViewResp save(@RequestBody ViewReq viewReq,
HttpServletRequest request, HttpServletResponse response) {
public DataSetResp save(@RequestBody DataSetReq dataSetReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewService.save(viewReq, user);
return dataSetService.save(dataSetReq, user);
}
@PutMapping
public ViewResp update(@RequestBody ViewReq viewReq,
HttpServletRequest request, HttpServletResponse response) {
public DataSetResp update(@RequestBody DataSetReq dataSetReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewService.update(viewReq, user);
return dataSetService.update(dataSetReq, user);
}
@GetMapping("/{id}")
public ViewResp getView(@PathVariable("id") Long id) {
return viewService.getView(id);
public DataSetResp getDataSet(@PathVariable("id") Long id) {
return dataSetService.getDataSet(id);
}
@GetMapping("/getViewList")
public List<ViewResp> getViewList(@RequestParam("domainId") Long domainId) {
@GetMapping("/getDataSetList")
public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDomainId(domainId);
return viewService.getViewList(metaFilter);
return dataSetService.getDataSetList(metaFilter);
}
@DeleteMapping("/{id}")
public Boolean delete(@PathVariable("id") Long id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
viewService.delete(id, user);
dataSetService.delete(id, user);
return true;
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java
View File

@@ -7,12 +7,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.service.SchemaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
@@ -34,8 +34,8 @@ public class SchemaController {
private SchemaService schemaService;
@PostMapping
public List<ViewSchemaResp> fetchViewSchema(@RequestBody ViewFilterReq filter) {
return schemaService.fetchViewSchema(filter);
public List<DataSetSchemaResp> fetchDataSetSchema(@RequestBody DataSetFilterReq filter) {
return schemaService.fetchDataSetSchema(filter);
}
@GetMapping("/domain/list")

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ViewQueryApiController.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/DataSetQueryApiController.java
View File

@@ -2,12 +2,10 @@ package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.ViewService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -15,22 +13,25 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class ViewQueryApiController {
public class DataSetQueryApiController {
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@Autowired
private QueryService queryService;
@PostMapping("/view")
public Object queryByView(@RequestBody QueryViewReq queryViewReq,
@PostMapping("/dataSet")
public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
SemanticQueryReq queryReq = viewService.convert(queryViewReq);
SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq);
return queryService.queryByReq(queryReq, user);
}

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java Normal file
View File

@@ -0,0 +1,32 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
import java.util.Map;
public interface DataSetService {
DataSetResp save(DataSetReq dataSetReq, User user);
DataSetResp update(DataSetReq dataSetReq, User user);
DataSetResp getDataSet(Long id);
List<DataSetResp> getDataSetList(MetaFilter metaFilter);
void delete(Long id, User user);
Map<Long, List<Long>> getModelIdToDataSetIds(List<Long> dataSetIds);
List<DataSetResp> getDataSets(User user);
List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId);
SemanticQueryReq convert(QueryDataSetReq queryDataSetReq);
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java
View File

@@ -12,9 +12,9 @@ public interface KnowledgeService {
List<S2Term> getTerms(String text);
List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds);
List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> dataSetIds);
List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> viewIds);
List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> dataSetIds);
void updateSemanticKnowledge(List<DictWord> natures);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java
View File

@@ -6,6 +6,6 @@ import java.util.List;
public interface MetaEmbeddingService {
List<RetrieveQueryResult> retrieveQuery(List<Long> viewIds, RetrieveQuery retrieveQuery, int num);
List<RetrieveQueryResult> retrieveQuery(List<Long> dataSetIds, RetrieveQuery retrieveQuery, int num);
}

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
@@ -17,15 +17,15 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import java.util.List;
import java.util.concurrent.ExecutionException;
public interface SchemaService {
List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter);
List<DataSetSchemaResp> fetchDataSetSchema(DataSetFilterReq filter);
List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds);
@@ -39,11 +39,11 @@ public interface SchemaService {
List<ModelResp> getModelList(User user, AuthType authType, Long domainId);
List<ViewResp> getViewList(Long domainId);
List<DataSetResp> getDataSetList(Long domainId);
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException;
List<ItemResp> getDomainViewTree();
List<ItemResp> getDomainDataSetTree();
}

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java
View File

@@ -1,32 +0,0 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
import java.util.Map;
public interface ViewService {
ViewResp save(ViewReq viewReq, User user);
ViewResp update(ViewReq viewReq, User user);
ViewResp getView(Long id);
List<ViewResp> getViewList(MetaFilter metaFilter);
void delete(Long id, User user);
Map<Long, List<Long>> getModelIdToViewIds(List<Long> viewIds);
List<ViewResp> getViews(User user);
List<ViewResp> getViewsInheritAuth(User user, Long domainId);
SemanticQueryReq convert(QueryViewReq queryViewReq);
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java
View File

@@ -24,7 +24,7 @@ import java.util.List;
@Service
public class CanvasServiceImpl {
private CanvasRepository viewInfoRepository;
private CanvasRepository canvasRepository;
private ModelService modelService;
@@ -32,16 +32,16 @@ public class CanvasServiceImpl {
private MetricService metricService;
public CanvasServiceImpl(CanvasRepository viewInfoRepository, ModelService modelService,
public CanvasServiceImpl(CanvasRepository canvasRepository, ModelService modelService,
MetricService metricService, DimensionService dimensionService) {
this.viewInfoRepository = viewInfoRepository;
this.canvasRepository = canvasRepository;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.modelService = modelService;
}
public List<CanvasDO> getCanvasList(Long domainId) {
return viewInfoRepository.getCanvasList(domainId);
return canvasRepository.getCanvasList(domainId);
}
public List<CanvasSchemaResp> getCanvasSchema(Long domainId, User user) {
@@ -70,20 +70,20 @@ public class CanvasServiceImpl {
viewInfoDO.setCreatedBy(user.getName());
viewInfoDO.setUpdatedAt(new Date());
viewInfoDO.setUpdatedBy(user.getName());
viewInfoRepository.createCanvas(viewInfoDO);
canvasRepository.createCanvas(viewInfoDO);
return viewInfoDO;
}
Long id = canvasReq.getId();
CanvasDO viewInfoDO = viewInfoRepository.getCanvasById(id);
CanvasDO viewInfoDO = canvasRepository.getCanvasById(id);
BeanUtils.copyProperties(canvasReq, viewInfoDO);
viewInfoDO.setUpdatedAt(new Date());
viewInfoDO.setUpdatedBy(user.getName());
viewInfoRepository.updateCanvas(viewInfoDO);
canvasRepository.updateCanvas(viewInfoDO);
return viewInfoDO;
}
public void deleteCanvas(Long id) {
viewInfoRepository.deleteCanvas(id);
canvasRepository.deleteCanvas(id);
}
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java
View File

@@ -25,7 +25,7 @@ import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;
@@ -45,16 +45,16 @@ public class CatalogImpl implements Catalog {
private final DimensionService dimensionService;
private final MetricService metricService;
private final ModelRelaService modelRelaService;
private final ViewService viewService;
private final DataSetService dataSetService;
private final SchemaService schemaService;
public CatalogImpl(DatabaseService databaseService, SchemaService schemaService,
ModelService modelService, DimensionService dimensionService, ViewService viewService,
ModelService modelService, DimensionService dimensionService, DataSetService dataSetService,
MetricService metricService, ModelRelaService modelRelaService) {
this.databaseService = databaseService;
this.modelService = modelService;
this.dimensionService = dimensionService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.metricService = metricService;
this.modelRelaService = modelRelaService;
this.schemaService = schemaService;

186
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetImpl.java
View File

@@ -13,23 +13,23 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
@@ -50,10 +50,10 @@ import java.util.function.Function;
import java.util.stream.Collectors;
@Service
public class ViewServiceImpl
extends ServiceImpl<ViewDOMapper, ViewDO> implements ViewService {
public class DataSetImpl
extends ServiceImpl<DataSetDOMapper, DataSetDO> implements DataSetService {
protected final Cache<MetaFilter, List<ViewResp>> viewSchemaCache =
protected final Cache<MetaFilter, List<DataSetResp>> dataSetSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
@Autowired
@@ -68,154 +68,156 @@ public class ViewServiceImpl
private MetricService metricService;
@Override
public ViewResp save(ViewReq viewReq, User user) {
viewReq.createdBy(user.getName());
ViewDO viewDO = convert(viewReq);
viewDO.setStatus(StatusEnum.ONLINE.getCode());
ViewResp viewResp = convert(viewDO);
conflictCheck(viewResp);
save(viewDO);
return viewResp;
public DataSetResp save(DataSetReq dataSetReq, User user) {
dataSetReq.createdBy(user.getName());
DataSetDO dataSetDO = convert(dataSetReq);
dataSetDO.setStatus(StatusEnum.ONLINE.getCode());
DataSetResp dataSetResp = convert(dataSetDO);
conflictCheck(dataSetResp);
save(dataSetDO);
return dataSetResp;
}
@Override
public ViewResp update(ViewReq viewReq, User user) {
viewReq.updatedBy(user.getName());
ViewDO viewDO = convert(viewReq);
ViewResp viewResp = convert(viewDO);
conflictCheck(viewResp);
updateById(viewDO);
return viewResp;
public DataSetResp update(DataSetReq dataSetReq, User user) {
dataSetReq.updatedBy(user.getName());
DataSetDO dataSetDO = convert(dataSetReq);
DataSetResp dataSetResp = convert(dataSetDO);
conflictCheck(dataSetResp);
updateById(dataSetDO);
return dataSetResp;
}
@Override
public ViewResp getView(Long id) {
ViewDO viewDO = getById(id);
return convert(viewDO);
public DataSetResp getDataSet(Long id) {
DataSetDO dataSetDO = getById(id);
return convert(dataSetDO);
}
@Override
public List<ViewResp> getViewList(MetaFilter metaFilter) {
QueryWrapper<ViewDO> wrapper = new QueryWrapper<>();
public List<DataSetResp> getDataSetList(MetaFilter metaFilter) {
QueryWrapper<DataSetDO> wrapper = new QueryWrapper<>();
if (metaFilter.getDomainId() != null) {
wrapper.lambda().eq(ViewDO::getDomainId, metaFilter.getDomainId());
wrapper.lambda().eq(DataSetDO::getDomainId, metaFilter.getDomainId());
}
if (!CollectionUtils.isEmpty(metaFilter.getIds())) {
wrapper.lambda().in(ViewDO::getId, metaFilter.getIds());
wrapper.lambda().in(DataSetDO::getId, metaFilter.getIds());
}
if (metaFilter.getStatus() != null) {
wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus());
wrapper.lambda().eq(DataSetDO::getStatus, metaFilter.getStatus());
}
wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode());
wrapper.lambda().ne(DataSetDO::getStatus, StatusEnum.DELETED.getCode());
return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
}
@Override
public void delete(Long id, User user) {
ViewDO viewDO = getById(id);
viewDO.setStatus(StatusEnum.DELETED.getCode());
viewDO.setUpdatedBy(user.getName());
viewDO.setUpdatedAt(new Date());
updateById(viewDO);
DataSetDO dataSetDO = getById(id);
dataSetDO.setStatus(StatusEnum.DELETED.getCode());
dataSetDO.setUpdatedBy(user.getName());
dataSetDO.setUpdatedAt(new Date());
updateById(dataSetDO);
}
@Override
public List<ViewResp> getViews(User user) {
List<ViewResp> viewResps = getViewList(new MetaFilter());
return getViewFilterByAuth(viewResps, user);
public List<DataSetResp> getDataSets(User user) {
List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter());
return getDataSetFilterByAuth(dataSetResps, user);
}
@Override
public List<ViewResp> getViewsInheritAuth(User user, Long domainId) {
List<ViewResp> viewResps = getViewList(new MetaFilter());
List<ViewResp> inheritAuthFormDomain = getViewFilterByDomainAuth(viewResps, user);
Set<ViewResp> viewRespSet = new HashSet<>(inheritAuthFormDomain);
List<ViewResp> viewFilterByAuth = getViewFilterByAuth(viewResps, user);
viewRespSet.addAll(viewFilterByAuth);
public List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId) {
List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter());
List<DataSetResp> inheritAuthFormDomain = getDataSetFilterByDomainAuth(dataSetResps, user);
Set<DataSetResp> dataSetRespSet = new HashSet<>(inheritAuthFormDomain);
List<DataSetResp> dataSetFilterByAuth = getDataSetFilterByAuth(dataSetResps, user);
dataSetRespSet.addAll(dataSetFilterByAuth);
if (domainId != null && domainId > 0) {
viewRespSet = viewRespSet.stream().filter(modelResp ->
dataSetRespSet = dataSetRespSet.stream().filter(modelResp ->
modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet());
}
return viewRespSet.stream().sorted(Comparator.comparingLong(ViewResp::getId))
return dataSetRespSet.stream().sorted(Comparator.comparingLong(DataSetResp::getId))
.collect(Collectors.toList());
}
private List<ViewResp> getViewFilterByAuth(List<ViewResp> viewResps, User user) {
return viewResps.stream()
.filter(viewResp -> checkAdminPermission(user, viewResp))
private List<DataSetResp> getDataSetFilterByAuth(List<DataSetResp> dataSetResps, User user) {
return dataSetResps.stream()
.filter(dataSetResp -> checkAdminPermission(user, dataSetResp))
.collect(Collectors.toList());
}
private List<ViewResp> getViewFilterByDomainAuth(List<ViewResp> viewResps, User user) {
private List<DataSetResp> getDataSetFilterByDomainAuth(List<DataSetResp> dataSetResps, User user) {
Set<DomainResp> domainResps = domainService.getDomainAuthSet(user, AuthType.ADMIN);
if (CollectionUtils.isEmpty(domainResps)) {
return Lists.newArrayList();
}
Set<Long> domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet());
return viewResps.stream().filter(viewResp ->
domainIds.contains(viewResp.getDomainId())).collect(Collectors.toList());
return dataSetResps.stream().filter(dataSetResp ->
domainIds.contains(dataSetResp.getDomainId())).collect(Collectors.toList());
}
private ViewResp convert(ViewDO viewDO) {
ViewResp viewResp = new ViewResp();
BeanMapper.mapper(viewDO, viewResp);
viewResp.setViewDetail(JSONObject.parseObject(viewDO.getViewDetail(), ViewDetail.class));
if (viewDO.getQueryConfig() != null) {
viewResp.setQueryConfig(JSONObject.parseObject(viewDO.getQueryConfig(), QueryConfig.class));
private DataSetResp convert(DataSetDO dataSetDO) {
DataSetResp dataSetResp = new DataSetResp();
BeanMapper.mapper(dataSetDO, dataSetResp);
dataSetResp.setDataSetDetail(JSONObject.parseObject(dataSetDO.getDataSetDetail(), DataSetDetail.class));
if (dataSetDO.getQueryConfig() != null) {
dataSetResp.setQueryConfig(JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class));
}
viewResp.setAdmins(StringUtils.isBlank(viewDO.getAdmin())
? Lists.newArrayList() : Arrays.asList(viewDO.getAdmin().split(",")));
viewResp.setAdminOrgs(StringUtils.isBlank(viewDO.getAdminOrg())
? Lists.newArrayList() : Arrays.asList(viewDO.getAdminOrg().split(",")));
viewResp.setTypeEnum(TypeEnums.VIEW);
return viewResp;
dataSetResp.setAdmins(StringUtils.isBlank(dataSetDO.getAdmin())
? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdmin().split(",")));
dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg())
? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(",")));
dataSetResp.setTypeEnum(TypeEnums.DATASET);
return dataSetResp;
}
private ViewDO convert(ViewReq viewReq) {
ViewDO viewDO = new ViewDO();
BeanMapper.mapper(viewReq, viewDO);
viewDO.setViewDetail(JSONObject.toJSONString(viewReq.getViewDetail()));
viewDO.setQueryConfig(JSONObject.toJSONString(viewReq.getQueryConfig()));
return viewDO;
private DataSetDO convert(DataSetReq dataSetReq) {
DataSetDO dataSetDO = new DataSetDO();
BeanMapper.mapper(dataSetReq, dataSetDO);
dataSetDO.setDataSetDetail(JSONObject.toJSONString(dataSetReq.getDataSetDetail()));
dataSetDO.setQueryConfig(JSONObject.toJSONString(dataSetReq.getQueryConfig()));
return dataSetDO;
}
public SemanticQueryReq convert(QueryViewReq queryViewReq) {
public SemanticQueryReq convert(QueryDataSetReq queryDataSetReq) {
SemanticQueryReq queryReq = new QueryStructReq();
if (StringUtils.isNotBlank(queryViewReq.getSql())) {
if (StringUtils.isNotBlank(queryDataSetReq.getSql())) {
queryReq = new QuerySqlReq();
}
BeanUtils.copyProperties(queryViewReq, queryReq);
BeanUtils.copyProperties(queryDataSetReq, queryReq);
return queryReq;
}
public static boolean checkAdminPermission(User user, ViewResp viewResp) {
List<String> admins = viewResp.getAdmins();
public static boolean checkAdminPermission(User user, DataSetResp dataSetResp) {
List<String> admins = dataSetResp.getAdmins();
if (user.isSuperAdmin()) {
return true;
}
String userName = user.getName();
return admins.contains(userName) || viewResp.getCreatedBy().equals(userName);
return admins.contains(userName) || dataSetResp.getCreatedBy().equals(userName);
}
@Override
public Map<Long, List<Long>> getModelIdToViewIds(List<Long> viewIds) {
public Map<Long, List<Long>> getModelIdToDataSetIds(List<Long> dataSetIds) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
metaFilter.setIds(viewIds);
List<ViewResp> viewList = viewSchemaCache.getIfPresent(metaFilter);
if (CollectionUtils.isEmpty(viewList)) {
viewList = getViewList(metaFilter);
viewSchemaCache.put(metaFilter, viewList);
metaFilter.setIds(dataSetIds);
List<DataSetResp> dataSetList = dataSetSchemaCache.getIfPresent(metaFilter);
if (CollectionUtils.isEmpty(dataSetList)) {
dataSetList = getDataSetList(metaFilter);
dataSetSchemaCache.put(metaFilter, dataSetList);
}
return viewList.stream()
return dataSetList.stream()
.flatMap(
viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId())))
.collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList())));
dataSetResp -> dataSetResp.getAllModels().stream().map(modelId ->
Pair.of(modelId, dataSetResp.getId())))
.collect(Collectors.groupingBy(Pair::getLeft,
Collectors.mapping(Pair::getRight, Collectors.toList())));
}
private void conflictCheck(ViewResp viewResp) {
List<Long> allDimensionIds = viewResp.getAllDimensions();
List<Long> allMetricIds = viewResp.getAllMetrics();
private void conflictCheck(DataSetResp dataSetResp) {
List<Long> allDimensionIds = dataSetResp.getAllDimensions();
List<Long> allMetricIds = dataSetResp.getAllMetrics();
MetaFilter metaFilter = new MetaFilter();
if (!CollectionUtils.isEmpty(allDimensionIds)) {
metaFilter.setIds(allDimensionIds);

29
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -21,29 +21,23 @@ import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -51,6 +45,13 @@ import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@Slf4j
public class DimensionServiceImpl implements DimensionService {
@@ -66,7 +67,7 @@ public class DimensionServiceImpl implements DimensionService {
private ModelRelaService modelRelaService;
private ViewService viewService;
private DataSetService dataSetService;
@Autowired
private ApplicationEventPublisher eventPublisher;
@@ -77,13 +78,13 @@ public class DimensionServiceImpl implements DimensionService {
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
ViewService viewService) {
DataSetService dataSetService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
this.viewService = viewService;
this.dataSetService = dataSetService;
}
@Override
@@ -232,9 +233,9 @@ public class DimensionServiceImpl implements DimensionService {
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(dimensionResps, metaFilter.getFieldsDepend());
}
if (metaFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(metaFilter.getViewId());
return DimensionConverter.filterByView(dimensionResps, viewResp);
if (metaFilter.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(metaFilter.getDataSetId());
return DimensionConverter.filterByDataSet(dimensionResps, dataSetResp);
}
return dimensionResps;
}

20
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java
View File

@@ -10,12 +10,12 @@ import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
import com.tencent.supersonic.headless.api.pojo.request.DomainUpdateReq;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import com.tencent.supersonic.headless.server.persistence.repository.DomainRepository;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.DomainConvert;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
@@ -43,17 +43,17 @@ public class DomainServiceImpl implements DomainService {
private final DomainRepository domainRepository;
private final ModelService modelService;
private final UserService userService;
private final ViewService viewService;
private final DataSetService dataSetService;
public DomainServiceImpl(DomainRepository domainRepository,
@Lazy ModelService modelService,
UserService userService,
@Lazy ViewService viewService) {
@Lazy DataSetService dataSetService) {
this.domainRepository = domainRepository;
this.modelService = modelService;
this.userService = userService;
this.viewService = viewService;
this.dataSetService = dataSetService;
}
@Override
@@ -104,9 +104,9 @@ public class DomainServiceImpl implements DomainService {
List<Long> domainIds = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList());
domainWithAuthAll.addAll(getParentDomain(domainIds));
}
List<ViewResp> viewResps = viewService.getViews(user);
if (!CollectionUtils.isEmpty(viewResps)) {
List<Long> domainIds = viewResps.stream().map(ViewResp::getDomainId).collect(Collectors.toList());
List<DataSetResp> dataSetResps = dataSetService.getDataSets(user);
if (!CollectionUtils.isEmpty(dataSetResps)) {
List<Long> domainIds = dataSetResps.stream().map(DataSetResp::getDomainId).collect(Collectors.toList());
domainWithAuthAll.addAll(getParentDomain(domainIds));
}
return new ArrayList<>(domainWithAuthAll).stream()
@@ -125,7 +125,7 @@ public class DomainServiceImpl implements DomainService {
}
if (authTypeEnum.equals(AuthType.VISIBLE)) {
domainWithAuth = domainResps.stream()
.filter(domainResp -> checkViewerPermission(orgIds, user, domainResp))
.filter(domainResp -> checkDataSeterPermission(orgIds, user, domainResp))
.collect(Collectors.toList());
}
List<Long> domainIds = domainWithAuth.stream().map(DomainResp::getId)
@@ -253,7 +253,7 @@ public class DomainServiceImpl implements DomainService {
return false;
}
private boolean checkViewerPermission(Set<String> orgIds, User user, DomainResp domainDesc) {
private boolean checkDataSeterPermission(Set<String> orgIds, User user, DomainResp domainDesc) {
List<String> admins = domainDesc.getAdmins();
List<String> viewers = domainDesc.getViewers();
List<String> adminOrgs = domainDesc.getAdminOrgs();

28
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -21,10 +21,10 @@ import java.util.stream.Collectors;
@Slf4j
public class KnowledgeServiceImpl implements KnowledgeService {
private final ViewService viewService;
private final DataSetService dataSetService;
public KnowledgeServiceImpl(ViewService viewService) {
this.viewService = viewService;
public KnowledgeServiceImpl(DataSetService dataSetService) {
this.dataSetService = dataSetService;
}
@Override
@@ -69,25 +69,25 @@ public class KnowledgeServiceImpl implements KnowledgeService {
@Override
public List<S2Term> getTerms(String text) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>());
return HanlpHelper.getTerms(text, modelIdToViewIds);
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>());
return HanlpHelper.getTerms(text, modelIdToDataSetIds);
}
@Override
public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
return prefixSearchByModel(key, limit, modelIdToViewIds);
public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> dataSetIds) {
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds));
return prefixSearchByModel(key, limit, modelIdToDataSetIds);
}
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
Map<Long, List<Long>> modelIdToViewIds) {
return SearchService.prefixSearch(key, limit, modelIdToViewIds);
Map<Long, List<Long>> modelIdToDataSetIds) {
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds);
}
@Override
public List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> viewIds) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
return suffixSearchByModel(key, limit, modelIdToViewIds.keySet());
public List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> dataSetIds) {
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds));
return suffixSearchByModel(key, limit, modelIdToDataSetIds.keySet());
}
public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Set<Long> models) {

33
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java
View File

@@ -7,8 +7,14 @@ import com.tencent.supersonic.common.util.embedding.Retrieval;
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.MetaEmbeddingService;
import com.tencent.supersonic.headless.server.service.ViewService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -17,11 +23,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@@ -32,13 +33,13 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService {
private EmbeddingConfig embeddingConfig;
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@Override
public List<RetrieveQueryResult> retrieveQuery(List<Long> viewIds, RetrieveQuery retrieveQuery, int num) {
// viewIds->modelIds
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(viewIds);
Set<Long> allModels = modelIdToViewIds.keySet();
public List<RetrieveQueryResult> retrieveQuery(List<Long> dataSetIds, RetrieveQuery retrieveQuery, int num) {
// dataSetIds->modelIds
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(dataSetIds);
Set<Long> allModels = modelIdToDataSetIds.keySet();
if (CollectionUtils.isNotEmpty(allModels) && allModels.size() == 1) {
Map<String, String> filterCondition = new HashMap<>();
@@ -69,16 +70,16 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService {
}
return !allModels.contains(modelId);
});
//add viewId
//add dataSetId
retrievals = retrievals.stream().flatMap(retrieval -> {
Long modelId = Retrieval.getLongId(retrieval.getMetadata().get("modelId"));
List<Long> viewIdsByModelId = modelIdToViewIds.get(modelId);
if (!CollectionUtils.isEmpty(viewIdsByModelId)) {
List<Long> dataSetIdsByModelId = modelIdToDataSetIds.get(modelId);
if (!CollectionUtils.isEmpty(dataSetIdsByModelId)) {
Set<Retrieval> result = new HashSet<>();
for (Long viewId : viewIdsByModelId) {
for (Long dataSetId : dataSetIdsByModelId) {
Retrieval retrievalNew = new Retrieval();
BeanUtils.copyProperties(retrieval, retrievalNew);
retrievalNew.getMetadata().putIfAbsent("viewId", viewId + Constants.UNDERLINE);
retrievalNew.getMetadata().putIfAbsent("dataSetId", dataSetId + Constants.UNDERLINE);
result.add(retrievalNew);
}
return result.stream();

27
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -29,10 +29,10 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
@@ -43,13 +43,19 @@ import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -60,11 +66,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Slf4j
@@ -80,7 +81,7 @@ public class MetricServiceImpl implements MetricService {
private CollectService collectService;
private ViewService viewService;
private DataSetService dataSetService;
private ApplicationEventPublisher eventPublisher;
@@ -88,7 +89,7 @@ public class MetricServiceImpl implements MetricService {
ModelService modelService,
ChatGptHelper chatGptHelper,
CollectService collectService,
ViewService viewService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService) {
this.metricRepository = metricRepository;
@@ -96,7 +97,7 @@ public class MetricServiceImpl implements MetricService {
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
this.collectService = collectService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.dimensionService = dimensionService;
}
@@ -232,9 +233,9 @@ public class MetricServiceImpl implements MetricService {
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(metricResps, metaFilter.getFieldsDepend());
}
if (metaFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(metaFilter.getViewId());
return MetricConverter.filterByView(metricResps, viewResp);
if (metaFilter.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(metaFilter.getDataSetId());
return MetricConverter.filterByDataSet(metricResps, dataSetResp);
}
return metricResps;
}

20
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -25,7 +25,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -37,7 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.ModelConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
@@ -76,7 +76,7 @@ public class ModelServiceImpl implements ModelService {
private UserService userService;
private ViewService viewService;
private DataSetService dataSetService;
private DateInfoRepository dateInfoRepository;
@@ -86,7 +86,7 @@ public class ModelServiceImpl implements ModelService {
@Lazy MetricService metricService,
DomainService domainService,
UserService userService,
ViewService viewService,
DataSetService dataSetService,
DateInfoRepository dateInfoRepository) {
this.modelRepository = modelRepository;
this.databaseService = databaseService;
@@ -94,7 +94,7 @@ public class ModelServiceImpl implements ModelService {
this.metricService = metricService;
this.domainService = domainService;
this.userService = userService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.dateInfoRepository = dateInfoRepository;
}
@@ -145,9 +145,9 @@ public class ModelServiceImpl implements ModelService {
ModelFilter modelFilter = new ModelFilter();
BeanUtils.copyProperties(metaFilter, modelFilter);
List<ModelResp> modelResps = ModelConverter.convertList(modelRepository.getModelList(modelFilter));
if (modelFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(modelFilter.getViewId());
return modelResps.stream().filter(modelResp -> viewResp.getAllModels().contains(modelResp.getId()))
if (modelFilter.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(modelFilter.getDataSetId());
return modelResps.stream().filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId()))
.collect(Collectors.toList());
}
return modelResps;
@@ -329,7 +329,7 @@ public class ModelServiceImpl implements ModelService {
}
if (authTypeEnum.equals(AuthType.VISIBLE)) {
modelWithAuth = modelResps.stream()
.filter(domainResp -> checkViewerPermission(orgIds, user, domainResp))
.filter(domainResp -> checkDataSeterPermission(orgIds, user, domainResp))
.collect(Collectors.toList());
}
return modelWithAuth;
@@ -456,7 +456,7 @@ public class ModelServiceImpl implements ModelService {
return false;
}
public static boolean checkViewerPermission(Set<String> orgIds, User user, ModelResp modelResp) {
public static boolean checkDataSeterPermission(Set<String> orgIds, User user, ModelResp modelResp) {
if (checkAdminPermission(orgIds, user, modelResp)) {
return true;
}

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -51,17 +51,18 @@ import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
@Service
@@ -176,7 +177,7 @@ public class QueryServiceImpl implements QueryService {
queryStatement.setQueryParam(queryParam);
queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setViewId(queryStructReq.getViewId());
queryStatement.setDataSetId(queryStructReq.getDataSetId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
@@ -216,7 +217,7 @@ public class QueryServiceImpl implements QueryService {
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setViewId(semanticQueryReq.getViewId());
schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId());
schemaFilterReq.setModelIds(semanticQueryReq.getModelIds());
return schemaFilterReq;
}

146
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
@@ -30,8 +30,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -41,7 +41,7 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.StatUtils;
@@ -68,7 +68,7 @@ public class SchemaServiceImpl implements SchemaService {
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
protected final Cache<ViewFilterReq, List<ViewSchemaResp>> viewSchemaCache =
protected final Cache<DataSetFilterReq, List<DataSetSchemaResp>> dataSetSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
protected final Cache<SchemaFilterReq, SemanticSchemaResp> semanticSchemaCache =
@@ -79,7 +79,7 @@ public class SchemaServiceImpl implements SchemaService {
private final DimensionService dimensionService;
private final MetricService metricService;
private final DomainService domainService;
private final ViewService viewService;
private final DataSetService dataSetService;
private final ModelRelaService modelRelaService;
private final TagService tagService;
@@ -87,14 +87,14 @@ public class SchemaServiceImpl implements SchemaService {
DimensionService dimensionService,
MetricService metricService,
DomainService domainService,
ViewService viewService,
DataSetService dataSetService,
ModelRelaService modelRelaService,
StatUtils statUtils, TagService tagService) {
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.domainService = domainService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.modelRelaService = modelRelaService;
this.statUtils = statUtils;
this.tagService = tagService;
@@ -102,30 +102,30 @@ public class SchemaServiceImpl implements SchemaService {
@SneakyThrows
@Override
public List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewList = viewSchemaCache.getIfPresent(filter);
if (CollectionUtils.isEmpty(viewList)) {
viewList = buildViewSchema(filter);
viewSchemaCache.put(filter, viewList);
public List<DataSetSchemaResp> fetchDataSetSchema(DataSetFilterReq filter) {
List<DataSetSchemaResp> dataSetList = dataSetSchemaCache.getIfPresent(filter);
if (CollectionUtils.isEmpty(dataSetList)) {
dataSetList = buildDataSetSchema(filter);
dataSetSchemaCache.put(filter, dataSetList);
}
return viewList;
return dataSetList;
}
public ViewSchemaResp fetchViewSchema(Long viewId) {
if (viewId == null) {
public DataSetSchemaResp fetchDataSetSchema(Long dataSetId) {
if (dataSetId == null) {
return null;
}
return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst().orElse(null);
}
public List<ViewSchemaResp> buildViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewSchemaResps = new ArrayList<>();
List<Long> viewIds = filter.getViewIds();
public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
List<DataSetSchemaResp> dataSetSchemaResps = new ArrayList<>();
List<Long> dataSetIds = filter.getDataSetIds();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
metaFilter.setIds(viewIds);
List<ViewResp> viewResps = viewService.getViewList(metaFilter);
List<Long> modelIds = viewResps.stream().map(ViewResp::getAllModels)
metaFilter.setIds(dataSetIds);
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(metaFilter);
List<Long> modelIds = dataSetResps.stream().map(DataSetResp::getAllModels)
.flatMap(Collection::stream).collect(Collectors.toList());
metaFilter.setModelIds(modelIds);
metaFilter.setIds(Lists.newArrayList());
@@ -133,26 +133,26 @@ public class SchemaServiceImpl implements SchemaService {
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
metaFilter.setIds(modelIds);
List<ModelResp> modelResps = modelService.getModelList(metaFilter);
Map<Long, ViewResp> viewRespMap = getViewMap(viewResps);
for (Long viewId : viewRespMap.keySet()) {
ViewResp viewResp = viewRespMap.get(viewId);
if (viewResp == null || !StatusEnum.ONLINE.getCode().equals(viewResp.getStatus())) {
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
for (Long dataSetId : dataSetRespMap.keySet()) {
DataSetResp dataSetResp = dataSetRespMap.get(dataSetId);
if (dataSetResp == null || !StatusEnum.ONLINE.getCode().equals(dataSetResp.getStatus())) {
continue;
}
List<MetricSchemaResp> metricSchemaResps = MetricConverter.filterByView(metricResps, viewResp)
List<MetricSchemaResp> metricSchemaResps = MetricConverter.filterByDataSet(metricResps, dataSetResp)
.stream().map(this::convert).collect(Collectors.toList());
List<DimSchemaResp> dimSchemaResps = DimensionConverter.filterByView(dimensionResps, viewResp)
List<DimSchemaResp> dimSchemaResps = DimensionConverter.filterByDataSet(dimensionResps, dataSetResp)
.stream().map(this::convert).collect(Collectors.toList());
ViewSchemaResp viewSchemaResp = new ViewSchemaResp();
BeanUtils.copyProperties(viewResp, viewSchemaResp);
viewSchemaResp.setDimensions(dimSchemaResps);
viewSchemaResp.setMetrics(metricSchemaResps);
viewSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
viewResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
viewSchemaResps.add(viewSchemaResp);
DataSetSchemaResp dataSetSchemaResp = new DataSetSchemaResp();
BeanUtils.copyProperties(dataSetResp, dataSetSchemaResp);
dataSetSchemaResp.setDimensions(dimSchemaResps);
dataSetSchemaResp.setMetrics(metricSchemaResps);
dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
dataSetSchemaResps.add(dataSetSchemaResp);
}
fillStaticInfo(viewSchemaResps);
return viewSchemaResps;
fillStaticInfo(dataSetSchemaResps);
return dataSetSchemaResps;
}
public List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds) {
@@ -191,7 +191,7 @@ public class SchemaServiceImpl implements SchemaService {
}
private void fillCnt(List<ViewSchemaResp> viewSchemaResps, List<ItemUseResp> statInfos) {
private void fillCnt(List<DataSetSchemaResp> dataSetSchemaResps, List<ItemUseResp> statInfos) {
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
.collect(Collectors.toMap(
@@ -199,15 +199,15 @@ public class SchemaServiceImpl implements SchemaService {
itemUseInfo -> itemUseInfo,
(item1, item2) -> item1));
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
for (ViewSchemaResp viewSchemaResp : viewSchemaResps) {
fillDimCnt(viewSchemaResp, typeIdAndStatPair);
fillMetricCnt(viewSchemaResp, typeIdAndStatPair);
for (DataSetSchemaResp dataSetSchemaResp : dataSetSchemaResps) {
fillDimCnt(dataSetSchemaResp, typeIdAndStatPair);
fillMetricCnt(dataSetSchemaResp, typeIdAndStatPair);
}
}
private void fillMetricCnt(ViewSchemaResp viewSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = viewSchemaResp.getMetrics();
if (CollectionUtils.isEmpty(viewSchemaResp.getMetrics())) {
private void fillMetricCnt(DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = dataSetSchemaResp.getMetrics();
if (CollectionUtils.isEmpty(dataSetSchemaResp.getMetrics())) {
return;
}
@@ -220,12 +220,12 @@ public class SchemaServiceImpl implements SchemaService {
}
});
}
viewSchemaResp.setMetrics(metrics);
dataSetSchemaResp.setMetrics(metrics);
}
private void fillDimCnt(ViewSchemaResp viewSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = viewSchemaResp.getDimensions();
if (CollectionUtils.isEmpty(viewSchemaResp.getDimensions())) {
private void fillDimCnt(DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = dataSetSchemaResp.getDimensions();
if (CollectionUtils.isEmpty(dataSetSchemaResp.getDimensions())) {
return;
}
if (!CollectionUtils.isEmpty(dimensions)) {
@@ -237,7 +237,7 @@ public class SchemaServiceImpl implements SchemaService {
}
});
}
viewSchemaResp.setDimensions(dimensions);
dataSetSchemaResp.setDimensions(dimensions);
}
@Override
@@ -273,20 +273,20 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public List<ViewResp> getViewList(Long domainId) {
public List<DataSetResp> getDataSetList(Long domainId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDomainId(domainId);
return viewService.getViewList(metaFilter);
return dataSetService.getDataSetList(metaFilter);
}
public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) {
SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
semanticSchemaResp.setViewId(schemaFilterReq.getViewId());
semanticSchemaResp.setDataSetId(schemaFilterReq.getDataSetId());
semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds());
if (schemaFilterReq.getViewId() != null) {
ViewSchemaResp viewSchemaResp = fetchViewSchema(schemaFilterReq.getViewId());
BeanUtils.copyProperties(viewSchemaResp, semanticSchemaResp);
List<Long> modelIds = viewSchemaResp.getAllModels();
if (schemaFilterReq.getDataSetId() != null) {
DataSetSchemaResp dataSetSchemaResp = fetchDataSetSchema(schemaFilterReq.getDataSetId());
BeanUtils.copyProperties(dataSetSchemaResp, semanticSchemaResp);
List<Long> modelIds = dataSetSchemaResp.getAllModels();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(modelIds);
List<ModelResp> modelList = modelService.getModelList(metaFilter);
@@ -343,7 +343,7 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public List<ItemResp> getDomainViewTree() {
public List<ItemResp> getDomainDataSetTree() {
List<DomainResp> domainResps = domainService.getDomainList();
List<ItemResp> itemResps = domainResps.stream().map(domain ->
new ItemResp(domain.getId(), domain.getParentId(), domain.getName(), TypeEnums.DOMAIN))
@@ -357,36 +357,36 @@ public class SchemaServiceImpl implements SchemaService {
}
parentItem.getChildren().add(itemResp);
}
List<ViewResp> viewResps = viewService.getViewList(new MetaFilter());
for (ViewResp viewResp : viewResps) {
ItemResp itemResp = itemRespMap.get(viewResp.getDomainId());
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(new MetaFilter());
for (DataSetResp dataSetResp : dataSetResps) {
ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId());
if (itemResp != null) {
ItemResp view = new ItemResp(viewResp.getId(), viewResp.getDomainId(),
viewResp.getName(), TypeEnums.VIEW);
itemResp.getChildren().add(view);
ItemResp dataSet = new ItemResp(dataSetResp.getId(), dataSetResp.getDomainId(),
dataSetResp.getName(), TypeEnums.DATASET);
itemResp.getChildren().add(dataSet);
}
}
return itemResps.stream().filter(itemResp -> itemResp.getParentId() == 0)
.collect(Collectors.toList());
}
private void fillStaticInfo(List<ViewSchemaResp> viewSchemaResps) {
List<Long> viewIds = viewSchemaResps.stream()
.map(ViewSchemaResp::getId).collect(Collectors.toList());
private void fillStaticInfo(List<DataSetSchemaResp> dataSetSchemaResps) {
List<Long> dataSetIds = dataSetSchemaResps.stream()
.map(DataSetSchemaResp::getId).collect(Collectors.toList());
ItemUseReq itemUseReq = new ItemUseReq();
itemUseReq.setModelIds(viewIds);
itemUseReq.setModelIds(dataSetIds);
List<ItemUseResp> statInfos = getStatInfo(itemUseReq);
log.debug("statInfos:{}", statInfos);
fillCnt(viewSchemaResps, statInfos);
fillCnt(dataSetSchemaResps, statInfos);
}
private Map<Long, ViewResp> getViewMap(List<ViewResp> viewResps) {
if (CollectionUtils.isEmpty(viewResps)) {
private Map<Long, DataSetResp> getDataSetMap(List<DataSetResp> dataSetResps) {
if (CollectionUtils.isEmpty(dataSetResps)) {
return new HashMap<>();
}
return viewResps.stream().collect(
Collectors.toMap(ViewResp::getId, viewResp -> viewResp));
return dataSetResps.stream().collect(
Collectors.toMap(DataSetResp::getId, dataSetResp -> dataSetResp));
}
private DimSchemaResp convert(DimensionResp dimensionResp) {

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
@@ -79,10 +79,10 @@ public class DimensionConverter {
return dimensionResp;
}
public static List<DimensionResp> filterByView(List<DimensionResp> dimensionResps, ViewResp viewResp) {
public static List<DimensionResp> filterByDataSet(List<DimensionResp> dimensionResps, DataSetResp dataSetResp) {
return dimensionResps.stream().filter(dimensionResp ->
viewResp.getAllDimensions().contains(dimensionResp.getId())
|| viewResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
dataSetResp.getAllDimensions().contains(dimensionResp.getId())
|| dataSetResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
.collect(Collectors.toList());
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.springframework.beans.BeanUtils;
@@ -99,10 +99,10 @@ public class MetricConverter {
return metricResp;
}
public static List<MetricResp> filterByView(List<MetricResp> metricResps, ViewResp viewResp) {
public static List<MetricResp> filterByDataSet(List<MetricResp> metricResps, DataSetResp dataSetResp) {
return metricResps.stream().filter(metricResp ->
viewResp.getAllMetrics().contains(metricResp.getId())
|| viewResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
dataSetResp.getAllMetrics().contains(metricResp.getId())
|| dataSetResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
.collect(Collectors.toList());
}

31
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -16,7 +16,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -27,7 +26,17 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,14 +46,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@@ -103,7 +104,7 @@ public class QueryReqConverter {
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ViewQueryParam result = new ViewQueryParam();
DataSetQueryParam result = new DataSetQueryParam();
BeanUtils.copyProperties(querySQLReq, result);
result.setTables(tables);
@@ -118,17 +119,17 @@ public class QueryReqConverter {
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setViewId(querySQLReq.getViewId());
queryStructReq.setDataSetId(querySQLReq.getDataSetId());
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam();
convert(queryStructReq, queryParam);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setDataSetQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(querySQLReq.getViewId());
queryStatement.setDataSetId(querySQLReq.getDataSetId());
queryStatement.setEnableLimitWrapper(limitWrapper);
return queryStatement;
@@ -225,7 +226,7 @@ public class QueryReqConverter {
public void correctTableName(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + querySqlReq.getViewId());
Constants.TABLE_PREFIX + querySqlReq.getDataSetId());
querySqlReq.setSql(sql);
}
@@ -239,7 +240,7 @@ public class QueryReqConverter {
}
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
ViewQueryParam viewQueryParam) {
DataSetQueryParam viewQueryParam) {
String sql = viewQueryParam.getSql();
for (MetricTable metricTable : viewQueryParam.getTables()) {
List<String> measures = new ArrayList<>();

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -77,7 +77,7 @@ public class QueryStructUtils {
private List<Long> getDimensionIds(QueryStructReq queryStructReq) {
List<Long> dimensionIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setViewId(queryStructReq.getViewId());
metaFilter.setDataSetId(queryStructReq.getDataSetId());
List<DimensionResp> dimensions = catalog.getDimensions(metaFilter);
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
@@ -99,7 +99,7 @@ public class QueryStructUtils {
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
List<Long> metricIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setViewId(queryStructCmd.getViewId());
metaFilter.setDataSetId(queryStructCmd.getDataSetId());
List<MetricResp> metrics = catalog.getMetrics(metaFilter);
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -20,18 +20,18 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@@ -41,17 +41,13 @@ public class StatUtils {
private static final TransmittableThreadLocal<QueryStat> STATS = new TransmittableThreadLocal<>();
private final StatRepository statRepository;
private final SqlFilterUtils sqlFilterUtils;
private final ModelService modelService;
private final ObjectMapper objectMapper = new ObjectMapper();
public StatUtils(StatRepository statRepository,
SqlFilterUtils sqlFilterUtils,
ModelService modelService) {
SqlFilterUtils sqlFilterUtils) {
this.statRepository = statRepository;
this.sqlFilterUtils = sqlFilterUtils;
this.modelService = modelService;
}
public static QueryStat get() {
@@ -112,7 +108,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId(traceId)
.setViewId(queryTagReq.getViewId())
.setDataSetId(queryTagReq.getDataSetId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
@@ -150,7 +146,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId("")
.setUser(userName)
.setViewId(querySqlReq.getViewId())
.setDataSetId(querySqlReq.getDataSetId())
.setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(querySqlReq.toString())
@@ -180,7 +176,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId(traceId)
.setViewId(queryStructReq.getViewId())
.setDataSetId(queryStructReq.getDataSetId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())

17
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java
View File

@@ -12,11 +12,8 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
@@ -24,6 +21,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Component
@Slf4j
public class TagReqConverter {
@@ -61,7 +62,7 @@ public class TagReqConverter {
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//.build ParseSqlReq
ViewQueryParam result = new ViewQueryParam();
DataSetQueryParam result = new DataSetQueryParam();
BeanUtils.copyProperties(querySqlReq, result);
result.setTables(tables);
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
@@ -72,15 +73,15 @@ public class TagReqConverter {
}
//.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setViewId(querySqlReq.getViewId());
queryStructReq.setDataSetId(querySqlReq.getDataSetId());
queryStructReq.setQueryType(QueryType.TAG);
QueryParam queryParam = new QueryParam();
convert(queryTagReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setDataSetQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(queryTagReq.getViewId());
queryStatement.setDataSetId(queryTagReq.getDataSetId());
queryStatement.setEnableLimitWrapper(limitWrapper);
}
return queryStatement;

8
headless/server/src/main/resources/mapper/StatMapper.xml
View File

@@ -9,7 +9,7 @@
<id column="id" property="id"/>
<result column="trace_id" property="traceId"/>
<result column="model_id" property="modelId"/>
<result column="view_id" property="viewId"/>
<result column="data_set_id" property="dataSetId"/>
<result column="user" property="user"/>
<result column="created_at" property="createdAt"/>
<result column="query_type" property="queryType"/>
@@ -43,13 +43,13 @@
<insert id="createRecord">
insert into s2_query_stat_info
(
trace_id, model_id, view_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
trace_id, model_id, data_set_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols,
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key, query_opt_mode
)
values
(
#{traceId}, #{modelId}, #{viewId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{traceId}, #{modelId}, #{dataSetId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}, #{queryOptMode}
)
@@ -61,7 +61,7 @@
from s2_query_stat_info
<where>
<if test="startTime != null">
and start_time >= #{startTime}
and start_time >= #{startTime}String.valueOf(queryFilter.getValue())
</if>
<if test="modelId != null">
and model_id = #{modelId}

4
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -23,7 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.DataSetImpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import java.util.HashMap;
import org.junit.jupiter.api.Assertions;
@@ -66,7 +66,7 @@ public class MetricServiceImplTest {
ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
ViewService viewService = Mockito.mock(ViewServiceImpl.class);
DataSetService viewService = Mockito.mock(DataSetImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, viewService,
eventPublisher, dimensionService);

2
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java
View File

@@ -75,7 +75,7 @@ class ModelServiceImplTest {
DomainService domainService = Mockito.mock(DomainService.class);
UserService userService = Mockito.mock(UserService.class);
DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
ViewService viewService = Mockito.mock(ViewService.class);
DataSetService viewService = Mockito.mock(DataSetService.class);
return new ModelServiceImpl(modelRepository, databaseService,
dimensionService, metricService, domainService, userService,
viewService, dateInfoRepository);