diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java similarity index 94% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java index fd71a4847..bf127623d 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java @@ -12,9 +12,9 @@ import java.util.Optional; import java.util.Set; @Data -public class ViewSchema { +public class DataSetSchema { - private SchemaElement view; + private SchemaElement dataSet; private Set metrics = new HashSet<>(); private Set dimensions = new HashSet<>(); private Set dimensionValues = new HashSet<>(); @@ -29,8 +29,8 @@ public class ViewSchema { case ENTITY: element = Optional.ofNullable(entity); break; - case VIEW: - element = Optional.of(view); + case DATASET: + element = Optional.of(dataSet); break; case METRIC: element = metrics.stream().filter(e -> e.getId() == elementID).findFirst(); @@ -61,8 +61,8 @@ public class ViewSchema { case ENTITY: element = Optional.ofNullable(entity); break; - case VIEW: - element = Optional.of(view); + case DATASET: + element = Optional.of(dataSet); break; case METRIC: element = metrics.stream().filter(e -> name.equals(e.getName())).findFirst(); diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java index f75ef845a..2a1b0f24e 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java @@ -9,25 +9,25 @@ import java.util.Set; public class SchemaMapInfo { - private Map> viewElementMatches = new HashMap<>(); + private Map> dataSetElementMatches = new HashMap<>(); - public Set getMatchedViewInfos() { - return viewElementMatches.keySet(); + public Set getMatchedDataSetInfos() { + return dataSetElementMatches.keySet(); } - public List getMatchedElements(Long view) { - return viewElementMatches.getOrDefault(view, Lists.newArrayList()); + public List getMatchedElements(Long dataSet) { + return dataSetElementMatches.getOrDefault(dataSet, Lists.newArrayList()); } - public Map> getViewElementMatches() { - return viewElementMatches; + public Map> getDataSetElementMatches() { + return dataSetElementMatches; } - public void setViewElementMatches(Map> viewElementMatches) { - this.viewElementMatches = viewElementMatches; + public void setDataSetElementMatches(Map> dataSetElementMatches) { + this.dataSetElementMatches = dataSetElementMatches; } - public void setMatchedElements(Long view, List elementMatches) { - viewElementMatches.put(view, elementMatches); + public void setMatchedElements(Long dataSet, List elementMatches) { + dataSetElementMatches.put(dataSet, elementMatches); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index da7992911..9d0f68f8f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -26,7 +26,7 @@ public class SemanticParseInfo { private Integer id; private String queryMode; - private SchemaElement view; + private SchemaElement dataSet; private Set metrics = new TreeSet<>(new SchemaNameLengthComparator()); private Set dimensions = new LinkedHashSet(); private SchemaElement entity; @@ -72,15 +72,11 @@ public class SemanticParseInfo { return metrics; } - public Long getViewId() { - if (view == null) { + public Long getDataSetId() { + if (dataSet == null) { return null; } - return view.getView(); - } - - public SchemaElement getModel() { - return view; + return dataSet.getDataSet(); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index 0d4678286..060f67d61 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -2,6 +2,8 @@ package com.tencent.supersonic.chat.api.pojo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import org.springframework.util.CollectionUtils; + import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -9,18 +11,17 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; public class SemanticSchema implements Serializable { - private List viewSchemaList; + private List dataSetSchemaList; - public SemanticSchema(List viewSchemaList) { - this.viewSchemaList = viewSchemaList; + public SemanticSchema(List dataSetSchemaList) { + this.dataSetSchemaList = dataSetSchemaList; } - public void add(ViewSchema schema) { - viewSchemaList.add(schema); + public void add(DataSetSchema schema) { + dataSetSchemaList.add(schema); } public SchemaElement getElement(SchemaElementType elementType, long elementID) { @@ -30,8 +31,8 @@ public class SemanticSchema implements Serializable { case ENTITY: element = getElementsById(elementID, getEntities()); break; - case VIEW: - element = getElementsById(elementID, getViews()); + case DATASET: + element = getElementsById(elementID, getDataSets()); break; case METRIC: element = getElementsById(elementID, getMetrics()); @@ -52,26 +53,26 @@ public class SemanticSchema implements Serializable { } } - public Map getViewIdToName() { - return viewSchemaList.stream() - .collect(Collectors.toMap(a -> a.getView().getId(), a -> a.getView().getName(), (k1, k2) -> k1)); + public Map getDataSetIdToName() { + return dataSetSchemaList.stream() + .collect(Collectors.toMap(a -> a.getDataSet().getId(), a -> a.getDataSet().getName(), (k1, k2) -> k1)); } public List getDimensionValues() { List dimensionValues = new ArrayList<>(); - viewSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues())); + dataSetSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues())); return dimensionValues; } public List getDimensions() { List dimensions = new ArrayList<>(); - viewSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions())); + dataSetSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions())); return dimensions; } - public List getDimensions(Long viewId) { + public List getDimensions(Long dataSetId) { List dimensions = getDimensions(); - return getElementsByViewId(viewId, dimensions); + return getElementsByDataSetId(dataSetId, dimensions); } public SchemaElement getDimension(Long id) { @@ -82,43 +83,43 @@ public class SemanticSchema implements Serializable { public List getTags() { List tags = new ArrayList<>(); - viewSchemaList.stream().forEach(d -> tags.addAll(d.getTags())); + dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTags())); return tags; } - public List getTags(Long viewId) { + public List getTags(Long dataSetId) { List tags = new ArrayList<>(); - viewSchemaList.stream().filter(schemaElement -> - viewId.equals(schemaElement.getView().getView())) + dataSetSchemaList.stream().filter(schemaElement -> + dataSetId.equals(schemaElement.getDataSet().getDataSet())) .forEach(d -> tags.addAll(d.getTags())); return tags; } public List getMetrics() { List metrics = new ArrayList<>(); - viewSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics())); + dataSetSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics())); return metrics; } - public List getMetrics(Long viewId) { + public List getMetrics(Long dataSetId) { List metrics = getMetrics(); - return getElementsByViewId(viewId, metrics); + return getElementsByDataSetId(dataSetId, metrics); } public List getEntities() { List entities = new ArrayList<>(); - viewSchemaList.stream().forEach(d -> entities.add(d.getEntity())); + dataSetSchemaList.stream().forEach(d -> entities.add(d.getEntity())); return entities; } - public List getEntities(Long viewId) { + public List getEntities(Long dataSetId) { List entities = getEntities(); - return getElementsByViewId(viewId, entities); + return getElementsByDataSetId(dataSetId, entities); } - private List getElementsByViewId(Long viewId, List elements) { + private List getElementsByDataSetId(Long dataSetId, List elements) { return elements.stream() - .filter(schemaElement -> viewId.equals(schemaElement.getView())) + .filter(schemaElement -> dataSetId.equals(schemaElement.getDataSet())) .collect(Collectors.toList()); } @@ -128,30 +129,30 @@ public class SemanticSchema implements Serializable { .findFirst(); } - public SchemaElement getView(Long viewId) { - List views = getViews(); - return getElementsById(viewId, views).orElse(null); + public SchemaElement getDataSet(Long dataSetId) { + List dataSets = getDataSets(); + return getElementsById(dataSetId, dataSets).orElse(null); } - public List getViews() { - List views = new ArrayList<>(); - viewSchemaList.stream().forEach(d -> views.add(d.getView())); - return views; + public List getDataSets() { + List dataSets = new ArrayList<>(); + dataSetSchemaList.stream().forEach(d -> dataSets.add(d.getDataSet())); + return dataSets; } - public Map getBizNameToName(Long viewId) { + public Map getBizNameToName(Long dataSetId) { List allElements = new ArrayList<>(); - allElements.addAll(getDimensions(viewId)); - allElements.addAll(getMetrics(viewId)); + allElements.addAll(getDimensions(dataSetId)); + allElements.addAll(getMetrics(dataSetId)); return allElements.stream() .collect(Collectors.toMap(SchemaElement::getBizName, SchemaElement::getName, (k1, k2) -> k1)); } - public Map getViewSchemaMap() { - if (CollectionUtils.isEmpty(viewSchemaList)) { + public Map getDataSetSchemaMap() { + if (CollectionUtils.isEmpty(dataSetSchemaList)) { return new HashMap<>(); } - return viewSchemaList.stream().collect(Collectors.toMap(viewSchema - -> viewSchema.getView().getView(), viewSchema -> viewSchema)); + return dataSetSchemaList.stream().collect(Collectors.toMap(dataSetSchema + -> dataSetSchema.getDataSet().getDataSet(), dataSetSchema -> dataSetSchema)); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java index 48457a279..f06c51531 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java @@ -13,7 +13,7 @@ public class PluginQueryReq { private String type; - private String view; + private String dataSet; private String pattern; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java index 25adb87a5..03e946136 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java @@ -7,7 +7,7 @@ import lombok.Data; public class QueryReq { private String queryText; private Integer chatId; - private Long viewId; + private Long dataSetId; private User user; private QueryFilters queryFilters; private boolean saveAnswer = true; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java index 7c5c6c9f2..7bcaec731 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java @@ -18,7 +18,7 @@ public class SimilarQueryReq { private String queryText; - private Long viewId; + private Long dataSetId; private Integer agentId; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ViewInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataSetInfo.java similarity index 74% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ViewInfo.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataSetInfo.java index d772b3ced..1433f051a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ViewInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataSetInfo.java @@ -6,7 +6,7 @@ import java.io.Serializable; import java.util.List; @Data -public class ViewInfo extends DataInfo implements Serializable { +public class DataSetInfo extends DataInfo implements Serializable { private List words; private String primaryKey; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java index dd6c07195..f261a9840 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java @@ -8,7 +8,7 @@ import java.util.List; @Data public class EntityInfo { - private ViewInfo viewInfo = new ViewInfo(); + private DataSetInfo dataSetInfo = new DataSetInfo(); private List dimensions = new ArrayList<>(); private List metrics = new ArrayList<>(); private String entityId; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java index 7a723c56f..f32e3a460 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java @@ -65,16 +65,16 @@ public class Agent extends RecordInfo { .collect(Collectors.toList()); } - public Set getViewIds() { - return getViewIds(null); + public Set getDataSetIds() { + return getDataSetIds(null); } - public Set getViewIds(AgentToolType agentToolType) { + public Set getDataSetIds(AgentToolType agentToolType) { List commonAgentTools = getParserTools(agentToolType); if (CollectionUtils.isEmpty(commonAgentTools)) { return new HashSet<>(); } - return commonAgentTools.stream().map(NL2SQLTool::getViewIds) + return commonAgentTools.stream().map(NL2SQLTool::getDataSetIds) .filter(modelIds -> !CollectionUtils.isEmpty(modelIds)) .flatMap(Collection::stream) .collect(Collectors.toSet()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java index eff5f11d0..4758b4422 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java @@ -12,6 +12,6 @@ import java.util.List; @AllArgsConstructor public class NL2SQLTool extends AgentTool { - protected List viewIds; + protected List dataSetIds; } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java index 45c6a7323..687e9fece 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java @@ -15,7 +15,7 @@ public class RuleParserTool extends NL2SQLTool { private List queryTypes; public boolean isContainsAllModel() { - return CollectionUtils.isNotEmpty(viewIds) && viewIds.contains(-1L); + return CollectionUtils.isNotEmpty(dataSetIds) && dataSetIds.contains(-1L); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java index bf530e3ad..0c9078a4c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java @@ -45,7 +45,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { public abstract void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo); - protected Map getFieldNameMap(QueryContext queryContext, Long viewId) { + protected Map getFieldNameMap(QueryContext queryContext, Long dataSetId) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); @@ -55,7 +55,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { // support fieldName and field alias Map result = dbAllFields.stream() - .filter(entry -> viewId.equals(entry.getView())) + .filter(entry -> dataSetId.equals(entry.getDataSet())) .flatMap(schemaElement -> { Set elements = new HashSet<>(); elements.add(schemaElement.getName()); @@ -109,8 +109,8 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { protected void addAggregateToMetric(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { //add aggregate to all metric String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - Long viewId = semanticParseInfo.getView().getView(); - List metrics = getMetricElements(queryContext, viewId); + Long dataSetId = semanticParseInfo.getDataSet().getDataSet(); + List metrics = getMetricElements(queryContext, dataSetId); Map metricToAggregate = metrics.stream() .map(schemaElement -> { @@ -135,13 +135,13 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql); } - protected List getMetricElements(QueryContext queryContext, Long viewId) { + protected List getMetricElements(QueryContext queryContext, Long dataSetId) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - return semanticSchema.getMetrics(viewId); + return semanticSchema.getMetrics(dataSetId); } - protected Set getDimensions(Long viewId, SemanticSchema semanticSchema) { - Set dimensions = semanticSchema.getDimensions(viewId).stream() + protected Set getDimensions(Long dataSetId, SemanticSchema semanticSchema) { + Set dimensions = semanticSchema.getDimensions(dataSetId).stream() .flatMap( schemaElement -> { Set elements = new HashSet<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java index 0b0f7c688..12d891b6c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java @@ -8,12 +8,13 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import java.util.List; import java.util.Objects; import java.util.Set; @@ -37,11 +38,12 @@ public class GroupByCorrector extends BaseSemanticCorrector { } private Boolean needAddGroupBy(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Long viewId = semanticParseInfo.getViewId(); - ViewService viewService = ContextUtils.getBean(ViewService.class); + Long dataSetId = semanticParseInfo.getDataSetId(); + DataSetService dataSetService = ContextUtils.getBean(DataSetService.class); ModelService modelService = ContextUtils.getBean(ModelService.class); - ViewResp viewResp = viewService.getView(viewId); - List modelIds = viewResp.getViewDetail().getViewModelConfigs().stream().map(config -> config.getId()) + DataSetResp dataSetResp = dataSetService.getDataSet(dataSetId); + List modelIds = dataSetResp.getDataSetDetail() + .getDataSetModelConfigs().stream().map(DataSetModelConfig::getId) .collect(Collectors.toList()); MetaFilter metaFilter = new MetaFilter(); metaFilter.setIds(modelIds); @@ -64,7 +66,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { return false; } //add alias field name - Set dimensions = getDimensions(viewId, semanticSchema); + Set dimensions = getDimensions(dataSetId, semanticSchema); List selectFields = SqlSelectHelper.getSelectFields(correctS2SQL); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) { return false; @@ -81,13 +83,13 @@ public class GroupByCorrector extends BaseSemanticCorrector { } private void addGroupByFields(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Long viewId = semanticParseInfo.getViewId(); + Long dataSetId = semanticParseInfo.getDataSetId(); //add dimension group by SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectS2SQL(); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); //add alias field name - Set dimensions = getDimensions(viewId, semanticSchema); + Set dimensions = getDimensions(dataSetId, semanticSchema); List selectFields = SqlSelectHelper.getSelectFields(correctS2SQL); List aggregateFields = SqlSelectHelper.getAggregateFields(correctS2SQL); Set groupByFields = selectFields.stream() diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java index e09d68ce2..02b2d381b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java @@ -39,11 +39,11 @@ public class HavingCorrector extends BaseSemanticCorrector { } private void addHaving(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Long viewId = semanticParseInfo.getView().getView(); + Long dataSet = semanticParseInfo.getDataSet().getDataSet(); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - Set metrics = semanticSchema.getMetrics(viewId).stream() + Set metrics = semanticSchema.getMetrics(dataSet).stream() .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); if (CollectionUtils.isEmpty(metrics)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java index ac9b70f18..b21829fa1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java @@ -16,15 +16,16 @@ import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; /** * Perform schema corrections on the Schema information in S2SQL. @@ -62,7 +63,7 @@ public class SchemaCorrector extends BaseSemanticCorrector { } private void correctFieldName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Map fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getViewId()); + Map fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getDataSetId()); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String sql = SqlReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap); sqlInfo.setCorrectS2SQL(sql); @@ -125,7 +126,7 @@ public class SchemaCorrector extends BaseSemanticCorrector { } List linkingValues = getLinkingValues(semanticParseInfo); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - Set dimensions = getDimensions(semanticParseInfo.getViewId(), semanticSchema); + Set dimensions = getDimensions(semanticParseInfo.getDataSetId(), semanticSchema); if (CollectionUtils.isEmpty(linkingValues)) { linkingValues = new ArrayList<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java index 07ef29067..ecc589275 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java @@ -67,7 +67,7 @@ public class WhereCorrector extends BaseSemanticCorrector { List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) { Pair startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, - semanticParseInfo.getViewId(), semanticParseInfo.getQueryType()); + semanticParseInfo.getDataSetId(), semanticParseInfo.getQueryType()); if (StringUtils.isNotBlank(startEndDate.getLeft()) && StringUtils.isNotBlank(startEndDate.getRight())) { correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); @@ -101,8 +101,8 @@ public class WhereCorrector extends BaseSemanticCorrector { private void updateFieldValueByTechName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - Long viewId = semanticParseInfo.getViewId(); - List dimensions = semanticSchema.getDimensions(viewId); + Long dataSetId = semanticParseInfo.getDataSetId(); + List dimensions = semanticSchema.getDimensions(dataSetId); if (CollectionUtils.isEmpty(dimensions)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java index 3a79e29b0..2d6b001e4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -26,7 +26,7 @@ public abstract class BaseMapper implements SchemaMapper { String simpleName = this.getClass().getSimpleName(); long startTime = System.currentTimeMillis(); - log.debug("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getViewElementMatches()); + log.debug("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getDataSetElementMatches()); try { doMap(queryContext); @@ -35,13 +35,14 @@ public abstract class BaseMapper implements SchemaMapper { } long cost = System.currentTimeMillis() - startTime; - log.debug("after {},cost:{},mapInfo:{}", simpleName, cost, queryContext.getMapInfo().getViewElementMatches()); + log.debug("after {},cost:{},mapInfo:{}", simpleName, cost, + queryContext.getMapInfo().getDataSetElementMatches()); } public abstract void doMap(QueryContext queryContext); public void addToSchemaMap(SchemaMapInfo schemaMap, Long modelId, SchemaElementMatch newElementMatch) { - Map> modelElementMatches = schemaMap.getViewElementMatches(); + Map> modelElementMatches = schemaMap.getDataSetElementMatches(); List schemaElementMatches = modelElementMatches.putIfAbsent(modelId, new ArrayList<>()); if (schemaElementMatches == null) { schemaElementMatches = modelElementMatches.get(modelId); @@ -67,14 +68,14 @@ public abstract class BaseMapper implements SchemaMapper { } } - public SchemaElement getSchemaElement(Long viewId, SchemaElementType elementType, Long elementID, + public SchemaElement getSchemaElement(Long dataSetId, SchemaElementType elementType, Long elementID, SemanticSchema semanticSchema) { SchemaElement element = new SchemaElement(); - ViewSchema viewSchema = semanticSchema.getViewSchemaMap().get(viewId); - if (Objects.isNull(viewSchema)) { + DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId); + if (Objects.isNull(dataSetSchema)) { return null; } - SchemaElement elementDb = viewSchema.getElement(elementType, elementID); + SchemaElement elementDb = dataSetSchema.getElement(elementType, elementID); if (Objects.isNull(elementDb)) { log.info("element is null, elementType:{},elementID:{}", elementType, elementID); return null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java index 7bc762246..5970d98bf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java @@ -28,22 +28,22 @@ public abstract class BaseMatchStrategy implements MatchStrategy { @Override public Map> match(QueryContext queryContext, List terms, - Set detectViewIds) { + Set detectDataSetIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; } - log.debug("terms:{},,detectViewIds:{}", terms, detectViewIds); + log.debug("terms:{},,detectDataSetIds:{}", terms, detectDataSetIds); - List detects = detect(queryContext, terms, detectViewIds); + List detects = detect(queryContext, terms, detectDataSetIds); Map> result = new HashMap<>(); result.put(MatchText.builder().regText(text).detectSegment(text).build(), detects); return result; } - public List detect(QueryContext queryContext, List terms, Set detectViewIds) { + public List detect(QueryContext queryContext, List terms, Set detectDataSetIds) { Map regOffsetToLength = getRegOffsetToLength(terms); String text = queryContext.getQueryText(); Set results = new HashSet<>(); @@ -58,16 +58,16 @@ public abstract class BaseMatchStrategy implements MatchStrategy { if (index <= text.length()) { String detectSegment = text.substring(startIndex, index).trim(); detectSegments.add(detectSegment); - detectByStep(queryContext, results, detectViewIds, detectSegment, offset); + detectByStep(queryContext, results, detectDataSetIds, detectSegment, offset); } } startIndex = mapperHelper.getStepIndex(regOffsetToLength, startIndex); } - detectByBatch(queryContext, results, detectViewIds, detectSegments); + detectByBatch(queryContext, results, detectDataSetIds, detectSegments); return new ArrayList<>(results); } - protected void detectByBatch(QueryContext queryContext, Set results, Set detectViewIds, + protected void detectByBatch(QueryContext queryContext, Set results, Set detectDataSetIds, Set detectSegments) { return; } @@ -104,9 +104,9 @@ public abstract class BaseMatchStrategy implements MatchStrategy { } public List getMatches(QueryContext queryContext, List terms) { - Set viewIds = mapperHelper.getViewIds(queryContext.getViewId(), queryContext.getAgent()); - terms = filterByViewId(terms, viewIds); - Map> matchResult = match(queryContext, terms, viewIds); + Set dataSetIds = mapperHelper.getDataSetIds(queryContext.getDataSetId(), queryContext.getAgent()); + terms = filterByDataSetId(terms, dataSetIds); + Map> matchResult = match(queryContext, terms, dataSetIds); List matches = new ArrayList<>(); if (Objects.isNull(matchResult)) { return matches; @@ -121,17 +121,17 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return matches; } - public List filterByViewId(List terms, Set viewIds) { + public List filterByDataSetId(List terms, Set dataSetIds) { logTerms(terms); - if (CollectionUtils.isNotEmpty(viewIds)) { + if (CollectionUtils.isNotEmpty(dataSetIds)) { terms = terms.stream().filter(term -> { - Long viewId = NatureHelper.getViewId(term.getNature().toString()); - if (Objects.nonNull(viewId)) { - return viewIds.contains(viewId); + Long dataSetId = NatureHelper.getDataSetId(term.getNature().toString()); + if (Objects.nonNull(dataSetId)) { + return dataSetIds.contains(dataSetId); } return false; }).collect(Collectors.toList()); - log.info("terms filter by viewId:{}", viewIds); + log.info("terms filter by dataSetId:{}", dataSetIds); logTerms(terms); } return terms; @@ -150,7 +150,7 @@ public abstract class BaseMatchStrategy implements MatchStrategy { public abstract String getMapKey(T a); - public abstract void detectByStep(QueryContext queryContext, Set existResults, Set detectViewIds, + public abstract void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, String detectSegment, int offset); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java index 3e20e65c1..2712bc371 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java @@ -37,9 +37,9 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy @Override public Map> match(QueryContext queryContext, List terms, - Set detectViewIds) { + Set detectDataSetIds) { this.allElements = getSchemaElements(queryContext); - return super.match(queryContext, terms, detectViewIds); + return super.match(queryContext, terms, detectDataSetIds); } @Override @@ -54,7 +54,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy + Constants.UNDERLINE + a.getSchemaElement().getName(); } - public void detectByStep(QueryContext queryContext, Set existResults, Set detectViewIds, + public void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, String detectSegment, int offset) { if (StringUtils.isBlank(detectSegment)) { return; @@ -70,9 +70,9 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy continue; } Set schemaElements = entry.getValue(); - if (!CollectionUtils.isEmpty(detectViewIds)) { + if (!CollectionUtils.isEmpty(detectDataSetIds)) { schemaElements = schemaElements.stream() - .filter(schemaElement -> detectViewIds.contains(schemaElement.getView())) + .filter(schemaElement -> detectDataSetIds.contains(schemaElement.getDataSet())) .collect(Collectors.toSet()); } for (SchemaElement schemaElement : schemaElements) { @@ -96,7 +96,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy Double metricDimensionThresholdConfig = optimizationConfig.getMetricDimensionThresholdConfig(); Double metricDimensionMinThresholdConfig = optimizationConfig.getMetricDimensionMinThresholdConfig(); - Map> modelElementMatches = queryContext.getMapInfo().getViewElementMatches(); + Map> modelElementMatches = queryContext.getMapInfo().getDataSetElementMatches(); boolean existElement = modelElementMatches.entrySet().stream().anyMatch(entry -> entry.getValue().size() >= 1); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java index 00d4a7b5b..2892defac 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java @@ -36,12 +36,12 @@ public class EmbeddingMapper extends BaseMapper { //2. build SchemaElementMatch by info for (EmbeddingResult matchResult : matchResults) { Long elementId = Retrieval.getLongId(matchResult.getId()); - Long viewId = Retrieval.getLongId(matchResult.getMetadata().get("viewId")); - if (Objects.isNull(viewId)) { + Long dataSetId = Retrieval.getLongId(matchResult.getMetadata().get("dataSetId")); + if (Objects.isNull(dataSetId)) { continue; } SchemaElementType elementType = SchemaElementType.valueOf(matchResult.getMetadata().get("type")); - SchemaElement schemaElement = getSchemaElement(viewId, elementType, elementId, + SchemaElement schemaElement = getSchemaElement(dataSetId, elementType, elementId, queryContext.getSemanticSchema()); if (schemaElement == null) { continue; @@ -54,7 +54,7 @@ public class EmbeddingMapper extends BaseMapper { .detectWord(matchResult.getDetectWord()) .build(); //3. add to mapInfo - addToSchemaMap(queryContext.getMapInfo(), viewId, schemaElementMatch); + addToSchemaMap(queryContext.getMapInfo(), dataSetId, schemaElementMatch); } } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java index a7ea4ef55..556802287 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java @@ -48,13 +48,13 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { } @Override - public void detectByStep(QueryContext queryContext, Set existResults, Set detectViewIds, + public void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, String detectSegment, int offset) { } @Override - protected void detectByBatch(QueryContext queryContext, Set results, Set detectViewIds, + protected void detectByBatch(QueryContext queryContext, Set results, Set detectDataSetIds, Set detectSegments) { List queryTextsList = detectSegments.stream() @@ -68,11 +68,11 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { optimizationConfig.getEmbeddingMapperBatch()); for (List queryTextsSub : queryTextsSubList) { - detectByQueryTextsSub(results, detectViewIds, queryTextsSub); + detectByQueryTextsSub(results, detectDataSetIds, queryTextsSub); } } - private void detectByQueryTextsSub(Set results, Set detectViewIds, + private void detectByQueryTextsSub(Set results, Set detectDataSetIds, List queryTextsSub) { int embeddingNumber = optimizationConfig.getEmbeddingMapperNumber(); Double distance = optimizationConfig.getEmbeddingMapperDistanceThreshold(); @@ -80,7 +80,7 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(queryTextsSub).build(); // step2. retrieveQuery by detectSegment List retrieveQueryResults = metaEmbeddingService.retrieveQuery( - new ArrayList<>(detectViewIds), retrieveQuery, embeddingNumber); + new ArrayList<>(detectDataSetIds), retrieveQuery, embeddingNumber); if (CollectionUtils.isEmpty(retrieveQueryResults)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java index 53d8560a4..645d17d6d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -24,12 +24,12 @@ public class EntityMapper extends BaseMapper { @Override public void doMap(QueryContext queryContext) { SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); - for (Long viewId : schemaMapInfo.getMatchedViewInfos()) { - List schemaElementMatchList = schemaMapInfo.getMatchedElements(viewId); + for (Long dataSetId : schemaMapInfo.getMatchedDataSetInfos()) { + List schemaElementMatchList = schemaMapInfo.getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(schemaElementMatchList)) { continue; } - SchemaElement entity = getEntity(viewId, queryContext); + SchemaElement entity = getEntity(dataSetId, queryContext); if (entity == null || entity.getId() == null) { continue; } @@ -65,9 +65,9 @@ public class EntityMapper extends BaseMapper { return false; } - private SchemaElement getEntity(Long viewId, QueryContext queryContext) { + private SchemaElement getEntity(Long dataSetId, QueryContext queryContext) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - ViewSchema modelSchema = semanticSchema.getViewSchemaMap().get(viewId); + DataSetSchema modelSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId); if (modelSchema != null && modelSchema.getEntity() != null) { return modelSchema.getEntity(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java index 62a5e8615..af9d62f56 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java @@ -39,15 +39,15 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { @Override public Map> match(QueryContext queryContext, List terms, - Set detectViewIds) { + Set detectDataSetIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; } - log.debug("retryCount:{},terms:{},,detectModelIds:{}", terms, detectViewIds); + log.debug("retryCount:{},terms:{},,detectModelIds:{}", terms, detectDataSetIds); - List detects = detect(queryContext, terms, detectViewIds); + List detects = detect(queryContext, terms, detectDataSetIds); Map> result = new HashMap<>(); result.put(MatchText.builder().regText(text).detectSegment(text).build(), detects); @@ -60,15 +60,15 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { && existResult.getDetectWord().length() < oneRoundResult.getDetectWord().length(); } - public void detectByStep(QueryContext queryContext, Set existResults, Set detectViewIds, + public void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, String detectSegment, int offset) { // step1. pre search Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize(); LinkedHashSet hanlpMapResults = knowledgeService.prefixSearch(detectSegment, - oneDetectionMaxSize, detectViewIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); + oneDetectionMaxSize, detectDataSetIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); // step2. suffix search LinkedHashSet suffixHanlpMapResults = knowledgeService.suffixSearch(detectSegment, - oneDetectionMaxSize, detectViewIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); + oneDetectionMaxSize, detectDataSetIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); hanlpMapResults.addAll(suffixHanlpMapResults); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java index 167af8d8f..a286d4c17 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java @@ -59,8 +59,8 @@ public class KeywordMapper extends BaseMapper { for (HanlpMapResult hanlpMapResult : mapResults) { for (String nature : hanlpMapResult.getNatures()) { - Long viewId = NatureHelper.getViewId(nature); - if (Objects.isNull(viewId)) { + Long dataSetId = NatureHelper.getDataSetId(nature); + if (Objects.isNull(dataSetId)) { continue; } SchemaElementType elementType = NatureHelper.convertToElementType(nature); @@ -68,7 +68,7 @@ public class KeywordMapper extends BaseMapper { continue; } Long elementID = NatureHelper.getElementID(nature); - SchemaElement element = getSchemaElement(viewId, elementType, + SchemaElement element = getSchemaElement(dataSetId, elementType, elementID, queryContext.getSemanticSchema()); if (element == null) { continue; @@ -85,7 +85,7 @@ public class KeywordMapper extends BaseMapper { .detectWord(hanlpMapResult.getDetectWord()) .build(); - addToSchemaMap(queryContext.getMapInfo(), viewId, schemaElementMatch); + addToSchemaMap(queryContext.getMapInfo(), dataSetId, schemaElementMatch); } } } @@ -106,12 +106,12 @@ public class KeywordMapper extends BaseMapper { .similarity(mapperHelper.getSimilarity(match.getDetectWord(), schemaElement.getName())) .build(); log.info("add to schema, elementMatch {}", schemaElementMatch); - addToSchemaMap(queryContext.getMapInfo(), schemaElement.getView(), schemaElementMatch); + addToSchemaMap(queryContext.getMapInfo(), schemaElement.getDataSet(), schemaElementMatch); } } private Set getRegElementSet(SchemaMapInfo schemaMap, SchemaElement schemaElement) { - List elements = schemaMap.getMatchedElements(schemaElement.getView()); + List elements = schemaMap.getMatchedElements(schemaElement.getDataSet()); if (CollectionUtils.isEmpty(elements)) { return new HashSet<>(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java index cd4fa5cc4..ceadebe81 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java @@ -62,7 +62,7 @@ public class MapperHelper { */ public boolean existDimensionValues(List natures) { for (String nature : natures) { - if (NatureHelper.isDimensionValueViewId(nature)) { + if (NatureHelper.isDimensionValueDataSetId(nature)) { return true; } } @@ -82,33 +82,33 @@ public class MapperHelper { detectSegment.length()); } - public Set getViewIds(Long viewId, Agent agent) { + public Set getDataSetIds(Long dataSetId, Agent agent) { - Set detectViewIds = new HashSet<>(); + Set detectDataSetIds = new HashSet<>(); if (Objects.nonNull(agent)) { - detectViewIds = agent.getViewIds(null); + detectDataSetIds = agent.getDataSetIds(); } //contains all - if (Agent.containsAllModel(detectViewIds)) { - if (Objects.nonNull(viewId) && viewId > 0) { + if (Agent.containsAllModel(detectDataSetIds)) { + if (Objects.nonNull(dataSetId) && dataSetId > 0) { Set result = new HashSet<>(); - result.add(viewId); + result.add(dataSetId); return result; } return new HashSet<>(); } - if (Objects.nonNull(detectViewIds)) { - detectViewIds = detectViewIds.stream().filter(entry -> entry > 0).collect(Collectors.toSet()); + if (Objects.nonNull(detectDataSetIds)) { + detectDataSetIds = detectDataSetIds.stream().filter(entry -> entry > 0).collect(Collectors.toSet()); } - if (Objects.nonNull(viewId) && viewId > 0 && Objects.nonNull(detectViewIds)) { - if (detectViewIds.contains(viewId)) { + if (Objects.nonNull(dataSetId) && dataSetId > 0 && Objects.nonNull(detectDataSetIds)) { + if (detectDataSetIds.contains(dataSetId)) { Set result = new HashSet<>(); - result.add(viewId); + result.add(dataSetId); return result; } } - return detectViewIds; + return detectDataSetIds; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java index 6987600de..b85f960c0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java @@ -13,6 +13,6 @@ import java.util.Set; */ public interface MatchStrategy { - Map> match(QueryContext queryContext, List terms, Set detectViewIds); + Map> match(QueryContext queryContext, List terms, Set detectDataSetIds); } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java index 20ba6743c..37d19776b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java @@ -27,13 +27,13 @@ public class QueryFilterMapper implements SchemaMapper { @Override public void map(QueryContext queryContext) { Agent agent = queryContext.getAgent(); - if (agent == null || CollectionUtils.isEmpty(agent.getViewIds())) { + if (agent == null || CollectionUtils.isEmpty(agent.getDataSetIds())) { return; } - if (Agent.containsAllModel(agent.getViewIds())) { + if (Agent.containsAllModel(agent.getDataSetIds())) { return; } - Set viewIds = agent.getViewIds(); + Set viewIds = agent.getDataSetIds(); SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); clearOtherSchemaElementMatch(viewIds, schemaMapInfo); for (Long viewId : viewIds) { @@ -47,7 +47,7 @@ public class QueryFilterMapper implements SchemaMapper { } private void clearOtherSchemaElementMatch(Set viewIds, SchemaMapInfo schemaMapInfo) { - for (Map.Entry> entry : schemaMapInfo.getViewElementMatches().entrySet()) { + for (Map.Entry> entry : schemaMapInfo.getDataSetElementMatches().entrySet()) { if (!viewIds.contains(entry.getKey())) { entry.getValue().clear(); } @@ -69,7 +69,7 @@ public class QueryFilterMapper implements SchemaMapper { .name(String.valueOf(filter.getValue())) .type(SchemaElementType.VALUE) .bizName(filter.getBizName()) - .view(viewId) + .dataSet(viewId) .build(); SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() .element(element) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java index b7dfd1e39..7af47379f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java @@ -32,7 +32,7 @@ public class SearchMatchStrategy extends BaseMatchStrategy { @Override public Map> match(QueryContext queryContext, List originals, - Set detectViewIds) { + Set detectDataSetIds) { String text = queryContext.getQueryText(); Map regOffsetToLength = getRegOffsetToLength(originals); @@ -57,9 +57,9 @@ public class SearchMatchStrategy extends BaseMatchStrategy { if (StringUtils.isNotEmpty(detectSegment)) { List hanlpMapResults = knowledgeService.prefixSearch(detectSegment, - SearchService.SEARCH_SIZE, detectViewIds); + SearchService.SEARCH_SIZE, detectDataSetIds); List suffixHanlpMapResults = knowledgeService.suffixSearch( - detectSegment, SEARCH_SIZE, detectViewIds); + detectSegment, SEARCH_SIZE, detectDataSetIds); hanlpMapResults.addAll(suffixHanlpMapResults); // remove entity name where search hanlpMapResults = hanlpMapResults.stream().filter(entry -> { @@ -93,7 +93,7 @@ public class SearchMatchStrategy extends BaseMatchStrategy { } @Override - public void detectByStep(QueryContext queryContext, Set existResults, Set detectViewIds, + public void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, String detectSegment, int offset) { } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java index a3f8f61e0..0e1119cb2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java @@ -38,12 +38,12 @@ public class JavaLLMProxy implements LLMProxy { return false; } - public LLMResp query2sql(LLMReq llmReq, Long viewId) { + public LLMResp query2sql(LLMReq llmReq, Long dataSetId) { SqlGeneration sqlGeneration = SqlGenerationFactory.get( SqlGenerationMode.getMode(llmReq.getSqlGenerationMode())); - String modelName = llmReq.getSchema().getViewName(); - LLMResp result = sqlGeneration.generation(llmReq, viewId); + String modelName = llmReq.getSchema().getDataSetName(); + LLMResp result = sqlGeneration.generation(llmReq, dataSetId); result.setQuery(llmReq.getQueryText()); result.setModelName(modelName); return result; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java index 0f47cf962..d11cefa77 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java @@ -15,7 +15,7 @@ public interface LLMProxy { boolean isSkip(QueryContext queryContext); - LLMResp query2sql(LLMReq llmReq, Long viewId); + LLMResp query2sql(LLMReq llmReq, Long dataSetId); FunctionResp requestFunction(FunctionReq functionReq); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java index 2cf77966f..2ed2b1bbe 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java @@ -48,10 +48,10 @@ public class PythonLLMProxy implements LLMProxy { return false; } - public LLMResp query2sql(LLMReq llmReq, Long viewId) { + public LLMResp query2sql(LLMReq llmReq, Long dataSetId) { long startTime = System.currentTimeMillis(); - log.info("requestLLM request, viewId:{},llmReq:{}", viewId, llmReq); - keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); + log.info("requestLLM request, dataSetId:{},llmReq:{}", dataSetId, llmReq); + keyPipelineLog.info("dataSetId:{},llmReq:{}", dataSetId, llmReq); try { LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java index de31c0ef6..c6ae7334d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java @@ -50,7 +50,7 @@ public class QueryTypeParser implements SemanticParser { return QueryType.ID; } //1. entity queryType - Long viewId = parseInfo.getViewId(); + Long dataSetId = parseInfo.getDataSetId(); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) { //If all the fields in the SELECT statement are of tag type. @@ -59,12 +59,12 @@ public class QueryTypeParser implements SemanticParser { .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(whereFields)) { - Set ids = semanticSchema.getEntities(viewId).stream().map(SchemaElement::getName) + Set ids = semanticSchema.getEntities(dataSetId).stream().map(SchemaElement::getName) .collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(ids) && ids.stream().anyMatch(whereFields::contains)) { return QueryType.ID; } - Set tags = semanticSchema.getTags(viewId).stream().map(SchemaElement::getName) + Set tags = semanticSchema.getTags(dataSetId).stream().map(SchemaElement::getName) .collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(tags) && tags.containsAll(whereFields)) { return QueryType.TAG; @@ -73,7 +73,7 @@ public class QueryTypeParser implements SemanticParser { } //2. metric queryType List selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getS2SQL()); - List metrics = semanticSchema.getMetrics(viewId); + List metrics = semanticSchema.getMetrics(dataSetId); if (CollectionUtils.isNotEmpty(metrics)) { Set metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); boolean containMetric = selectFields.stream().anyMatch(metricNameSet::contains); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java index 8ff3ef04a..c8abda334 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java @@ -55,13 +55,13 @@ public abstract class PluginParser implements SemanticParser { public void buildQuery(QueryContext queryContext, PluginRecallResult pluginRecallResult) { Plugin plugin = pluginRecallResult.getPlugin(); - Set viewIds = pluginRecallResult.getViewIds(); + Set dataSetIds = pluginRecallResult.getDataSetIds(); if (plugin.isContainsAllModel()) { - viewIds = Sets.newHashSet(-1L); + dataSetIds = Sets.newHashSet(-1L); } - for (Long viewId : viewIds) { + for (Long dataSetId : dataSetIds) { PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); - SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(viewId, plugin, + SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(dataSetId, plugin, queryContext, pluginRecallResult.getDistance()); semanticParseInfo.setQueryMode(pluginQuery.getQueryMode()); semanticParseInfo.setScore(pluginRecallResult.getScore()); @@ -74,19 +74,19 @@ public abstract class PluginParser implements SemanticParser { return PluginManager.getPluginAgentCanSupport(queryContext); } - protected SemanticParseInfo buildSemanticParseInfo(Long viewId, Plugin plugin, + protected SemanticParseInfo buildSemanticParseInfo(Long dataSetId, Plugin plugin, QueryContext queryContext, double distance) { - List schemaElementMatches = queryContext.getMapInfo().getMatchedElements(viewId); + List schemaElementMatches = queryContext.getMapInfo().getMatchedElements(dataSetId); QueryFilters queryFilters = queryContext.getQueryFilters(); - if (viewId == null && !CollectionUtils.isEmpty(plugin.getViewList())) { - viewId = plugin.getViewList().get(0); + if (dataSetId == null && !CollectionUtils.isEmpty(plugin.getDataSetList())) { + dataSetId = plugin.getDataSetList().get(0); } if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); } SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setElementMatches(schemaElementMatches); - semanticParseInfo.setView(queryContext.getSemanticSchema().getView(viewId)); + semanticParseInfo.setDataSet(queryContext.getSemanticSchema().getDataSet(dataSetId)); Map properties = new HashMap<>(); PluginParseResult pluginParseResult = new PluginParseResult(); pluginParseResult.setPlugin(plugin); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java index f0b7f1b4a..b7153d6cd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java @@ -57,15 +57,15 @@ public class EmbeddingRecallParser extends PluginParser { Pair> pair = PluginManager.resolve(plugin, queryContext); log.info("embedding plugin resolve: {}", pair); if (pair.getLeft()) { - Set viewList = pair.getRight(); - if (CollectionUtils.isEmpty(viewList)) { + Set dataSetList = pair.getRight(); + if (CollectionUtils.isEmpty(dataSetList)) { continue; } plugin.setParseMode(ParseMode.EMBEDDING_RECALL); double distance = embeddingRetrieval.getDistance(); double score = queryContext.getQueryText().length() * (1 - distance); return PluginRecallResult.builder() - .plugin(plugin).viewIds(viewList).score(score).distance(distance).build(); + .plugin(plugin).dataSetIds(dataSetList).score(score).distance(distance).build(); } } return null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java index 69f39ea73..7c8ba3fa0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java @@ -57,19 +57,19 @@ public class FunctionCallParser extends PluginParser { plugin.setParseMode(ParseMode.FUNCTION_CALL); Pair> pluginResolveResult = PluginManager.resolve(plugin, queryContext); if (pluginResolveResult.getLeft()) { - Set viewList = pluginResolveResult.getRight(); - if (CollectionUtils.isEmpty(viewList)) { + Set dataSetList = pluginResolveResult.getRight(); + if (CollectionUtils.isEmpty(dataSetList)) { return null; } double score = queryContext.getQueryText().length(); - return PluginRecallResult.builder().plugin(plugin).viewIds(viewList).score(score).build(); + return PluginRecallResult.builder().plugin(plugin).dataSetIds(dataSetList).score(score).build(); } return null; } public FunctionResp functionCall(QueryContext queryContext) { List pluginToFunctionCall = - getPluginToFunctionCall(queryContext.getViewId(), queryContext); + getPluginToFunctionCall(queryContext.getDataSetId(), queryContext); if (CollectionUtils.isEmpty(pluginToFunctionCall)) { log.info("function call parser, plugin is empty, skip"); return null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewMatchResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetMatchResult.java similarity index 81% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewMatchResult.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetMatchResult.java index 7d3acaba0..ccb65e7af 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewMatchResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetMatchResult.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import lombok.Data; @Data -public class ViewMatchResult { +public class DataSetMatchResult { private Integer count = 0; private double maxSimilarity; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetResolver.java similarity index 86% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetResolver.java index e3bb72f20..edde73b05 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetResolver.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.core.pojo.QueryContext; import java.util.Set; -public interface ViewResolver { +public interface DataSetResolver { Long resolve(QueryContext queryContext, Set restrictiveModels); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicDataSetResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicDataSetResolver.java new file mode 100644 index 000000000..efb8370f3 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicDataSetResolver.java @@ -0,0 +1,138 @@ +package com.tencent.supersonic.chat.core.parser.sql.llm; + +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.chat.core.query.SemanticQuery; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class HeuristicDataSetResolver implements DataSetResolver { + + protected static Long selectDataSetBySchemaElementMatchScore(Map dataSetQueryModes, + SchemaMapInfo schemaMap) { + //dataSet count priority + Long dataSetIdByDataSetCount = getDataSetIdByMatchDataSetScore(schemaMap); + if (Objects.nonNull(dataSetIdByDataSetCount)) { + log.info("selectDataSet by dataSet count:{}", dataSetIdByDataSetCount); + return dataSetIdByDataSetCount; + } + + Map dataSetTypeMap = getDataSetTypeMap(schemaMap); + if (dataSetTypeMap.size() == 1) { + Long dataSetSelect = new ArrayList<>(dataSetTypeMap.entrySet()).get(0).getKey(); + if (dataSetQueryModes.containsKey(dataSetSelect)) { + log.info("selectDataSet with only one DataSet [{}]", dataSetSelect); + return dataSetSelect; + } + } else { + Map.Entry maxDataSet = dataSetTypeMap.entrySet().stream() + .filter(entry -> dataSetQueryModes.containsKey(entry.getKey())) + .sorted((o1, o2) -> { + int difference = o2.getValue().getCount() - o1.getValue().getCount(); + if (difference == 0) { + return (int) ((o2.getValue().getMaxSimilarity() + - o1.getValue().getMaxSimilarity()) * 100); + } + return difference; + }).findFirst().orElse(null); + if (maxDataSet != null) { + log.info("selectDataSet with multiple DataSets [{}]", maxDataSet.getKey()); + return maxDataSet.getKey(); + } + } + return null; + } + + private static Long getDataSetIdByMatchDataSetScore(SchemaMapInfo schemaMap) { + Map> dataSetElementMatches = schemaMap.getDataSetElementMatches(); + // calculate dataSet match score, matched element gets 1.0 point, and inherit element gets 0.5 point + Map dataSetIdToDataSetScore = new HashMap<>(); + if (Objects.nonNull(dataSetElementMatches)) { + for (Entry> dataSetElementMatch : dataSetElementMatches.entrySet()) { + Long dataSetId = dataSetElementMatch.getKey(); + List dataSetMatchesScore = dataSetElementMatch.getValue().stream() + .filter(elementMatch -> elementMatch.getSimilarity() >= 1) + .filter(elementMatch -> SchemaElementType.DATASET.equals(elementMatch.getElement().getType())) + .map(elementMatch -> elementMatch.isInherited() ? 0.5 : 1.0).collect(Collectors.toList()); + + if (!CollectionUtils.isEmpty(dataSetMatchesScore)) { + // get sum of dataSet match score + double score = dataSetMatchesScore.stream().mapToDouble(Double::doubleValue).sum(); + dataSetIdToDataSetScore.put(dataSetId, score); + } + } + Entry maxDataSetScore = dataSetIdToDataSetScore.entrySet().stream() + .max(Comparator.comparingDouble(Entry::getValue)).orElse(null); + log.info("maxDataSetCount:{},dataSetIdToDataSetCount:{}", maxDataSetScore, dataSetIdToDataSetScore); + if (Objects.nonNull(maxDataSetScore)) { + return maxDataSetScore.getKey(); + } + } + return null; + } + + public static Map getDataSetTypeMap(SchemaMapInfo schemaMap) { + Map dataSetCount = new HashMap<>(); + for (Map.Entry> entry : schemaMap.getDataSetElementMatches().entrySet()) { + List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); + if (schemaElementMatches != null && schemaElementMatches.size() > 0) { + if (!dataSetCount.containsKey(entry.getKey())) { + dataSetCount.put(entry.getKey(), new DataSetMatchResult()); + } + DataSetMatchResult dataSetMatchResult = dataSetCount.get(entry.getKey()); + Set schemaElementTypes = new HashSet<>(); + schemaElementMatches.stream() + .forEach(schemaElementMatch -> schemaElementTypes.add( + schemaElementMatch.getElement().getType())); + SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() + .sorted((o1, o2) -> + ((int) ((o2.getSimilarity() - o1.getSimilarity()) * 100)) + ).findFirst().orElse(null); + if (schemaElementMatchMax != null) { + dataSetMatchResult.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); + } + dataSetMatchResult.setCount(schemaElementTypes.size()); + + } + } + return dataSetCount; + } + + public Long resolve(QueryContext queryContext, Set agentDataSetIds) { + SchemaMapInfo mapInfo = queryContext.getMapInfo(); + Set matchedDataSets = mapInfo.getMatchedDataSetInfos(); + Long dataSetId = queryContext.getDataSetId(); + if (Objects.nonNull(dataSetId) && dataSetId > 0) { + if (CollectionUtils.isEmpty(agentDataSetIds) || agentDataSetIds.contains(dataSetId)) { + return dataSetId; + } + return null; + } + if (CollectionUtils.isNotEmpty(agentDataSetIds)) { + matchedDataSets.retainAll(agentDataSetIds); + } + Map dataSetQueryModes = new HashMap<>(); + for (Long dataSetIds : matchedDataSets) { + dataSetQueryModes.put(dataSetIds, null); + } + if (dataSetQueryModes.size() == 1) { + return dataSetQueryModes.keySet().stream().findFirst().get(); + } + return selectDataSetBySchemaElementMatchScore(dataSetQueryModes, mapInfo); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java deleted file mode 100644 index 366d0666e..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java +++ /dev/null @@ -1,138 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; - -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -public class HeuristicViewResolver implements ViewResolver { - - protected static Long selectViewBySchemaElementMatchScore(Map viewQueryModes, - SchemaMapInfo schemaMap) { - //view count priority - Long viewIdByViewCount = getViewIdByMatchViewScore(schemaMap); - if (Objects.nonNull(viewIdByViewCount)) { - log.info("selectView by view count:{}", viewIdByViewCount); - return viewIdByViewCount; - } - - Map viewTypeMap = getViewTypeMap(schemaMap); - if (viewTypeMap.size() == 1) { - Long viewSelect = new ArrayList<>(viewTypeMap.entrySet()).get(0).getKey(); - if (viewQueryModes.containsKey(viewSelect)) { - log.info("selectView with only one View [{}]", viewSelect); - return viewSelect; - } - } else { - Map.Entry maxView = viewTypeMap.entrySet().stream() - .filter(entry -> viewQueryModes.containsKey(entry.getKey())) - .sorted((o1, o2) -> { - int difference = o2.getValue().getCount() - o1.getValue().getCount(); - if (difference == 0) { - return (int) ((o2.getValue().getMaxSimilarity() - - o1.getValue().getMaxSimilarity()) * 100); - } - return difference; - }).findFirst().orElse(null); - if (maxView != null) { - log.info("selectView with multiple Views [{}]", maxView.getKey()); - return maxView.getKey(); - } - } - return null; - } - - private static Long getViewIdByMatchViewScore(SchemaMapInfo schemaMap) { - Map> viewElementMatches = schemaMap.getViewElementMatches(); - // calculate view match score, matched element gets 1.0 point, and inherit element gets 0.5 point - Map viewIdToViewScore = new HashMap<>(); - if (Objects.nonNull(viewElementMatches)) { - for (Entry> viewElementMatch : viewElementMatches.entrySet()) { - Long viewId = viewElementMatch.getKey(); - List viewMatchesScore = viewElementMatch.getValue().stream() - .filter(elementMatch -> elementMatch.getSimilarity() >= 1) - .filter(elementMatch -> SchemaElementType.VIEW.equals(elementMatch.getElement().getType())) - .map(elementMatch -> elementMatch.isInherited() ? 0.5 : 1.0).collect(Collectors.toList()); - - if (!CollectionUtils.isEmpty(viewMatchesScore)) { - // get sum of view match score - double score = viewMatchesScore.stream().mapToDouble(Double::doubleValue).sum(); - viewIdToViewScore.put(viewId, score); - } - } - Entry maxViewScore = viewIdToViewScore.entrySet().stream() - .max(Comparator.comparingDouble(Entry::getValue)).orElse(null); - log.info("maxViewCount:{},viewIdToViewCount:{}", maxViewScore, viewIdToViewScore); - if (Objects.nonNull(maxViewScore)) { - return maxViewScore.getKey(); - } - } - return null; - } - - public static Map getViewTypeMap(SchemaMapInfo schemaMap) { - Map viewCount = new HashMap<>(); - for (Map.Entry> entry : schemaMap.getViewElementMatches().entrySet()) { - List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); - if (schemaElementMatches != null && schemaElementMatches.size() > 0) { - if (!viewCount.containsKey(entry.getKey())) { - viewCount.put(entry.getKey(), new ViewMatchResult()); - } - ViewMatchResult viewMatchResult = viewCount.get(entry.getKey()); - Set schemaElementTypes = new HashSet<>(); - schemaElementMatches.stream() - .forEach(schemaElementMatch -> schemaElementTypes.add( - schemaElementMatch.getElement().getType())); - SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() - .sorted((o1, o2) -> - ((int) ((o2.getSimilarity() - o1.getSimilarity()) * 100)) - ).findFirst().orElse(null); - if (schemaElementMatchMax != null) { - viewMatchResult.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); - } - viewMatchResult.setCount(schemaElementTypes.size()); - - } - } - return viewCount; - } - - public Long resolve(QueryContext queryContext, Set agentViewIds) { - SchemaMapInfo mapInfo = queryContext.getMapInfo(); - Set matchedViews = mapInfo.getMatchedViewInfos(); - Long viewId = queryContext.getViewId(); - if (Objects.nonNull(viewId) && viewId > 0) { - if (CollectionUtils.isEmpty(agentViewIds) || agentViewIds.contains(viewId)) { - return viewId; - } - return null; - } - if (CollectionUtils.isNotEmpty(agentViewIds)) { - matchedViews.retainAll(agentViewIds); - } - Map viewQueryModes = new HashMap<>(); - for (Long viewIds : matchedViews) { - viewQueryModes.put(viewIds, null); - } - if (viewQueryModes.size() == 1) { - return viewQueryModes.keySet().stream().findFirst().get(); - } - return selectViewBySchemaElementMatchScore(viewQueryModes, mapInfo); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java index cde924968..18852d1a1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java @@ -22,7 +22,7 @@ import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -61,20 +61,20 @@ public class LLMRequestService { return false; } - public Long getViewId(QueryContext queryCtx) { + public Long getDataSetId(QueryContext queryCtx) { Agent agent = queryCtx.getAgent(); - Set agentViewIds = new HashSet<>(); + Set agentDataSetIds = new HashSet<>(); if (Objects.nonNull(agent)) { - agentViewIds = agent.getViewIds(AgentToolType.NL2SQL_LLM); + agentDataSetIds = agent.getDataSetIds(AgentToolType.NL2SQL_LLM); } - if (Agent.containsAllModel(agentViewIds)) { - agentViewIds = new HashSet<>(); + if (Agent.containsAllModel(agentDataSetIds)) { + agentDataSetIds = new HashSet<>(); } - ViewResolver viewResolver = ComponentFactory.getModelResolver(); - return viewResolver.resolve(queryCtx, agentViewIds); + DataSetResolver dataSetResolver = ComponentFactory.getModelResolver(); + return dataSetResolver.resolve(queryCtx, agentDataSetIds); } - public NL2SQLTool getParserTool(QueryContext queryCtx, Long viewId) { + public NL2SQLTool getParserTool(QueryContext queryCtx, Long dataSetId) { Agent agent = queryCtx.getAgent(); if (Objects.isNull(agent)) { return null; @@ -82,19 +82,19 @@ public class LLMRequestService { List commonAgentTools = agent.getParserTools(AgentToolType.NL2SQL_LLM); Optional llmParserTool = commonAgentTools.stream() .filter(tool -> { - List viewIds = tool.getViewIds(); - if (Agent.containsAllModel(new HashSet<>(viewIds))) { + List dataSetIds = tool.getDataSetIds(); + if (Agent.containsAllModel(new HashSet<>(dataSetIds))) { return true; } - return viewIds.contains(viewId); + return dataSetIds.contains(dataSetId); }) .findFirst(); return llmParserTool.orElse(null); } - public LLMReq getLlmReq(QueryContext queryCtx, Long viewId, + public LLMReq getLlmReq(QueryContext queryCtx, Long dataSetId, SemanticSchema semanticSchema, List linkingValues) { - Map viewIdToName = semanticSchema.getViewIdToName(); + Map dataSetIdToName = semanticSchema.getDataSetIdToName(); String queryText = queryCtx.getQueryText(); LLMReq llmReq = new LLMReq(); @@ -103,12 +103,12 @@ public class LLMRequestService { llmReq.setFilterCondition(filterCondition); LLMReq.LLMSchema llmSchema = new LLMReq.LLMSchema(); - llmSchema.setViewName(viewIdToName.get(viewId)); - llmSchema.setDomainName(viewIdToName.get(viewId)); + llmSchema.setDataSetName(dataSetIdToName.get(dataSetId)); + llmSchema.setDomainName(dataSetIdToName.get(dataSetId)); - List fieldNameList = getFieldNameList(queryCtx, viewId, llmParserConfig); + List fieldNameList = getFieldNameList(queryCtx, dataSetId, llmParserConfig); - String priorExts = getPriorExts(viewId, fieldNameList); + String priorExts = getPriorExts(dataSetId, fieldNameList); llmReq.setPriorExts(priorExts); fieldNameList.add(TimeDimensionEnum.DAY.getChName()); @@ -121,7 +121,7 @@ public class LLMRequestService { } llmReq.setLinking(linking); - String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, viewId); + String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, dataSetId); if (StringUtils.isEmpty(currentDate)) { currentDate = DateUtils.getBeforeDate(0); } @@ -130,28 +130,28 @@ public class LLMRequestService { return llmReq; } - public LLMResp requestLLM(LLMReq llmReq, Long viewId) { - return ComponentFactory.getLLMProxy().query2sql(llmReq, viewId); + public LLMResp requestLLM(LLMReq llmReq, Long dataSetId) { + return ComponentFactory.getLLMProxy().query2sql(llmReq, dataSetId); } - protected List getFieldNameList(QueryContext queryCtx, Long viewId, + protected List getFieldNameList(QueryContext queryCtx, Long dataSetId, LLMParserConfig llmParserConfig) { - Set results = getTopNFieldNames(queryCtx, viewId, llmParserConfig); + Set results = getTopNFieldNames(queryCtx, dataSetId, llmParserConfig); - Set fieldNameList = getMatchedFieldNames(queryCtx, viewId); + Set fieldNameList = getMatchedFieldNames(queryCtx, dataSetId); results.addAll(fieldNameList); return new ArrayList<>(results); } - private String getPriorExts(Long viewId, List fieldNameList) { + private String getPriorExts(Long dataSetId, List fieldNameList) { StringBuilder extraInfoSb = new StringBuilder(); - List viewSchemaResps = semanticInterpreter.fetchViewSchema( - Lists.newArrayList(viewId), true); - if (!CollectionUtils.isEmpty(viewSchemaResps)) { - ViewSchemaResp viewSchemaResp = viewSchemaResps.get(0); - Map fieldNameToDataFormatType = viewSchemaResp.getMetrics() + List dataSetSchemaResps = semanticInterpreter.fetchDataSetSchema( + Lists.newArrayList(dataSetId), true); + if (!CollectionUtils.isEmpty(dataSetSchemaResps)) { + DataSetSchemaResp dataSetSchemaResp = dataSetSchemaResps.get(0); + Map fieldNameToDataFormatType = dataSetSchemaResp.getMetrics() .stream().filter(metricSchemaResp -> Objects.nonNull(metricSchemaResp.getDataFormatType())) .flatMap(metricSchemaResp -> { Set> result = new HashSet<>(); @@ -179,9 +179,9 @@ public class LLMRequestService { return extraInfoSb.toString(); } - protected List getValueList(QueryContext queryCtx, Long viewId) { - Map itemIdToName = getItemIdToName(queryCtx, viewId); - List matchedElements = queryCtx.getMapInfo().getMatchedElements(viewId); + protected List getValueList(QueryContext queryCtx, Long dataSetId) { + Map itemIdToName = getItemIdToName(queryCtx, dataSetId); + List matchedElements = queryCtx.getMapInfo().getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(matchedElements)) { return new ArrayList<>(); } @@ -201,21 +201,21 @@ public class LLMRequestService { return new ArrayList<>(valueMatches); } - protected Map getItemIdToName(QueryContext queryCtx, Long viewId) { + protected Map getItemIdToName(QueryContext queryCtx, Long dataSetId) { SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); - return semanticSchema.getDimensions(viewId).stream() + return semanticSchema.getDimensions(dataSetId).stream() .collect(Collectors.toMap(SchemaElement::getId, SchemaElement::getName, (value1, value2) -> value2)); } - private Set getTopNFieldNames(QueryContext queryCtx, Long viewId, LLMParserConfig llmParserConfig) { + private Set getTopNFieldNames(QueryContext queryCtx, Long dataSetId, LLMParserConfig llmParserConfig) { SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); - Set results = semanticSchema.getDimensions(viewId).stream() + Set results = semanticSchema.getDimensions(dataSetId).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getDimensionTopN()) .map(entry -> entry.getName()) .collect(Collectors.toSet()); - Set metrics = semanticSchema.getMetrics(viewId).stream() + Set metrics = semanticSchema.getMetrics(dataSetId).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getMetricTopN()) .map(entry -> entry.getName()) @@ -225,9 +225,9 @@ public class LLMRequestService { return results; } - protected Set getMatchedFieldNames(QueryContext queryCtx, Long viewId) { - Map itemIdToName = getItemIdToName(queryCtx, viewId); - List matchedElements = queryCtx.getMapInfo().getMatchedElements(viewId); + protected Set getMatchedFieldNames(QueryContext queryCtx, Long dataSetId) { + Map itemIdToName = getItemIdToName(queryCtx, dataSetId); + List matchedElements = queryCtx.getMapInfo().getMatchedElements(dataSetId); if (CollectionUtils.isEmpty(matchedElements)) { return new HashSet<>(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java index bb2dc2afe..a9b3b0f84 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java @@ -28,9 +28,9 @@ public class LLMResponseService { } LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(LLMSqlQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); - parseInfo.setView(queryCtx.getSemanticSchema().getView(parseResult.getViewId())); + parseInfo.setDataSet(queryCtx.getSemanticSchema().getDataSet(parseResult.getDataSetId())); NL2SQLTool commonAgentTool = parseResult.getCommonAgentTool(); - parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getViewId())); + parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getDataSetId())); Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, parseResult); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java index d415b9473..de8e055af 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java @@ -29,21 +29,21 @@ public class LLMSqlParser implements SemanticParser { } try { //2.get modelId from queryCtx and chatCtx. - Long viewId = requestService.getViewId(queryCtx); - if (viewId == null) { + Long dataSetId = requestService.getDataSetId(queryCtx); + if (dataSetId == null) { return; } //3.get agent tool and determine whether to skip this parser. - NL2SQLTool commonAgentTool = requestService.getParserTool(queryCtx, viewId); + NL2SQLTool commonAgentTool = requestService.getParserTool(queryCtx, dataSetId); if (Objects.isNull(commonAgentTool)) { log.info("no tool in this agent, skip {}", LLMSqlParser.class); return; } //4.construct a request, call the API for the large model, and retrieve the results. - List linkingValues = requestService.getValueList(queryCtx, viewId); + List linkingValues = requestService.getValueList(queryCtx, dataSetId); SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); - LLMReq llmReq = requestService.getLlmReq(queryCtx, viewId, semanticSchema, linkingValues); - LLMResp llmResp = requestService.requestLLM(llmReq, viewId); + LLMReq llmReq = requestService.getLlmReq(queryCtx, dataSetId, semanticSchema, linkingValues); + LLMResp llmResp = requestService.requestLLM(llmReq, dataSetId); if (Objects.isNull(llmResp)) { return; @@ -52,7 +52,7 @@ public class LLMSqlParser implements SemanticParser { LLMResponseService responseService = ContextUtils.getBean(LLMResponseService.class); Map deduplicationSqlResp = responseService.getDeduplicationSqlResp(llmResp); ParseResult parseResult = ParseResult.builder() - .viewId(viewId) + .dataSetId(dataSetId) .commonAgentTool(commonAgentTool) .llmReq(llmReq) .llmResp(llmResp) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java index 1342bf0cd..11324108f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java @@ -43,9 +43,9 @@ public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, Long viewId) { + public LLMResp generation(LLMReq llmReq, Long dataSetId) { //1.retriever sqlExamples and generate exampleListPool - keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); + keyPipelineLog.info("dataSetId:{},llmReq:{}", dataSetId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java index a98486ae1..df35c6dfe 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java @@ -41,9 +41,9 @@ public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, Long viewId) { + public LLMResp generation(LLMReq llmReq, Long dataSetId) { //1.retriever sqlExamples - keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); + keyPipelineLog.info("dataSetId:{},llmReq:{}", dataSetId, llmReq); List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java index ef323bde0..d3866da51 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java @@ -18,7 +18,7 @@ import java.util.List; @NoArgsConstructor public class ParseResult { - private Long viewId; + private Long dataSetId; private LLMReq llmReq; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java index e897e76e0..48d266b49 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java @@ -12,9 +12,9 @@ public interface SqlGeneration { /*** * generate llmResp (sql, schemaLink, prompt, etc.) through LLMReq. * @param llmReq - * @param viewId + * @param dataSetId * @return */ - LLMResp generation(LLMReq llmReq, Long viewId); + LLMResp generation(LLMReq llmReq, Long dataSetId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java index 155ef0655..daf031825 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java @@ -96,7 +96,7 @@ public class SqlPromptGenerator { } public Pair transformQuestionPrompt(LLMReq llmReq) { - String modelName = llmReq.getSchema().getViewName(); + String modelName = llmReq.getSchema().getDataSetName(); List fieldNameList = llmReq.getSchema().getFieldNameList(); List linking = llmReq.getLinking(); String currentDate = llmReq.getCurrentDate(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java index a90af3086..5803f2811 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java @@ -40,9 +40,9 @@ public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, Long viewId) { + public LLMResp generation(LLMReq llmReq, Long dataSetId) { //1.retriever sqlExamples and generate exampleListPool - keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); + keyPipelineLog.info("dataSetId:{},llmReq:{}", dataSetId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java index 61fe24b60..2866f2772 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java @@ -40,8 +40,8 @@ public class TwoPassSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, Long viewId) { - keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); + public LLMResp generation(LLMReq llmReq, Long dataSetId) { + keyPipelineLog.info("dataSetId:{},llmReq:{}", dataSetId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java index 3fd10dfdd..f1f2533c5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java @@ -58,13 +58,13 @@ public class AgentCheckParser implements SemanticParser { } } } - if (CollectionUtils.isEmpty(tool.getViewIds())) { + if (CollectionUtils.isEmpty(tool.getDataSetIds())) { return true; } if (tool.isContainsAllModel()) { return false; } - return !tool.getViewIds().contains(query.getParseInfo().getViewId()); + return !tool.getDataSetIds().contains(query.getParseInfo().getDataSetId()); } return true; }); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java index 6d4f9cf74..cf6c1e824 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java @@ -39,7 +39,7 @@ public class ContextInheritParser implements SemanticParser { SchemaElementType.VALUE, Arrays.asList(SchemaElementType.VALUE, SchemaElementType.DIMENSION)), new AbstractMap.SimpleEntry<>(SchemaElementType.ENTITY, Arrays.asList(SchemaElementType.ENTITY)), new AbstractMap.SimpleEntry<>(SchemaElementType.TAG, Arrays.asList(SchemaElementType.TAG)), - new AbstractMap.SimpleEntry<>(SchemaElementType.VIEW, Arrays.asList(SchemaElementType.VIEW)), + new AbstractMap.SimpleEntry<>(SchemaElementType.DATASET, Arrays.asList(SchemaElementType.DATASET)), new AbstractMap.SimpleEntry<>(SchemaElementType.ID, Arrays.asList(SchemaElementType.ID)) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -48,12 +48,12 @@ public class ContextInheritParser implements SemanticParser { if (!shouldInherit(queryContext)) { return; } - Long viewId = getMatchedView(queryContext, chatContext); - if (viewId == null) { + Long dataSetId = getMatchedDataSet(queryContext, chatContext); + if (dataSetId == null) { return; } - List elementMatches = queryContext.getMapInfo().getMatchedElements(viewId); + List elementMatches = queryContext.getMapInfo().getMatchedElements(dataSetId); List matchesToInherit = new ArrayList<>(); for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) { @@ -70,17 +70,17 @@ public class ContextInheritParser implements SemanticParser { List queries = RuleSemanticQuery.resolve(elementMatches, queryContext); for (RuleSemanticQuery query : queries) { query.fillParseInfo(queryContext, chatContext); - if (existSameQuery(query.getParseInfo().getViewId(), query.getQueryMode(), queryContext)) { + if (existSameQuery(query.getParseInfo().getDataSetId(), query.getQueryMode(), queryContext)) { continue; } queryContext.getCandidateQueries().add(query); } } - private boolean existSameQuery(Long viewId, String queryMode, QueryContext queryContext) { + private boolean existSameQuery(Long dataSetId, String queryMode, QueryContext queryContext) { for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) { if (semanticQuery.getQueryMode().equals(queryMode) - && semanticQuery.getParseInfo().getViewId().equals(viewId)) { + && semanticQuery.getParseInfo().getDataSetId().equals(dataSetId)) { return true; } } @@ -109,16 +109,16 @@ public class ContextInheritParser implements SemanticParser { return metricModelQueries.size() == queryContext.getCandidateQueries().size(); } - protected Long getMatchedView(QueryContext queryContext, ChatContext chatContext) { - Long viewId = chatContext.getParseInfo().getViewId(); - if (viewId == null) { + protected Long getMatchedDataSet(QueryContext queryContext, ChatContext chatContext) { + Long dataSetId = chatContext.getParseInfo().getDataSetId(); + if (dataSetId == null) { return null; } - Set queryViews = queryContext.getMapInfo().getMatchedViewInfos(); - if (queryViews.contains(viewId)) { - return viewId; + Set queryDataSets = queryContext.getMapInfo().getMatchedDataSetInfos(); + if (queryDataSets.contains(dataSetId)) { + return dataSetId; } - return viewId; + return dataSetId; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java index 48ce387b3..6dae880a5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java @@ -29,8 +29,8 @@ public class RuleSqlParser implements SemanticParser { public void parse(QueryContext queryContext, ChatContext chatContext) { SchemaMapInfo mapInfo = queryContext.getMapInfo(); // iterate all schemaElementMatches to resolve query mode - for (Long viewId : mapInfo.getMatchedViewInfos()) { - List elementMatches = mapInfo.getMatchedElements(viewId); + for (Long dataSetId : mapInfo.getMatchedDataSetInfos()) { + List elementMatches = mapInfo.getMatchedElements(dataSetId); List queries = RuleSemanticQuery.resolve(elementMatches, queryContext); for (RuleSemanticQuery query : queries) { query.fillParseInfo(queryContext, chatContext); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java index 8f578986d..06fc40579 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java @@ -20,7 +20,7 @@ public class Plugin extends RecordInfo { */ private String type; - private List viewList = Lists.newArrayList(); + private List dataSetList = Lists.newArrayList(); /** * description, for parsing @@ -52,7 +52,7 @@ public class Plugin extends RecordInfo { } public boolean isContainsAllModel() { - return CollectionUtils.isNotEmpty(viewList) && viewList.contains(-1L); + return CollectionUtils.isNotEmpty(dataSetList) && dataSetList.contains(-1L); } public Long getDefaultMode() { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java index f9d315c14..576da5196 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java @@ -266,14 +266,14 @@ public class PluginManager { } private static Set getPluginMatchedModel(Plugin plugin, QueryContext queryContext) { - Set matchedViews = queryContext.getMapInfo().getMatchedViewInfos(); + Set matchedDataSets = queryContext.getMapInfo().getMatchedDataSetInfos(); if (plugin.isContainsAllModel()) { return Sets.newHashSet(plugin.getDefaultMode()); } - List modelIds = plugin.getViewList(); + List modelIds = plugin.getDataSetList(); Set pluginMatchedModel = Sets.newHashSet(); for (Long modelId : modelIds) { - if (matchedViews.contains(modelId)) { + if (matchedDataSets.contains(modelId)) { pluginMatchedModel.add(modelId); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java index 14d6225b1..91b257977 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java @@ -15,7 +15,7 @@ public class PluginRecallResult { private Plugin plugin; - private Set viewIds; + private Set dataSetIds; private double score; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java index 6c45f4b85..c8ca91334 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java @@ -30,7 +30,7 @@ public class QueryContext { private String queryText; private Integer chatId; - private Long viewId; + private Long dataSetId; private User user; private boolean saveAnswer = true; private Integer agentId; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 3a38c49b0..f8288bb3f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -49,7 +49,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { explainSqlReq = ExplainSqlReq.builder() .queryTypeEnum(QueryType.SQL) .queryReq(QueryReqBuilder.buildS2SQLReq( - sqlInfo.getCorrectS2SQL(), parseInfo.getViewId() + sqlInfo.getCorrectS2SQL(), parseInfo.getDataSetId() )) .build(); } else { @@ -84,7 +84,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } protected void convertBizNameToName(SemanticSchema semanticSchema, QueryStructReq queryStructReq) { - Map bizNameToName = semanticSchema.getBizNameToName(queryStructReq.getViewId()); + Map bizNameToName = semanticSchema.getBizNameToName(queryStructReq.getDataSetId()); bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap()); List orders = queryStructReq.getOrders(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java index 4745eb511..5dfe3c2ea 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java @@ -36,7 +36,7 @@ public class LLMReq { private String domainName; - private String viewName; + private String dataSetName; private List fieldNameList; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java index 940082617..1581860ba 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java @@ -42,7 +42,7 @@ public class LLMSqlQuery extends LLMSemanticQuery { long startTime = System.currentTimeMillis(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getViewId()); + QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getDataSetId()); SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java index c62b53fa8..17df377e6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java @@ -79,7 +79,7 @@ public abstract class PluginSemanticQuery extends BaseSemanticQuery { if (!CollectionUtils.isEmpty(webPage.getParamOptions()) && !CollectionUtils.isEmpty(elementValueMap)) { for (ParamOption paramOption : webPage.getParamOptions()) { if (paramOption.getModelId() != null - && !parseInfo.getViewId().equals(paramOption.getModelId())) { + && !parseInfo.getDataSetId().equals(paramOption.getModelId())) { continue; } paramOptions.add(paramOption); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java index 80e0c71a3..5f4dbadfa 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java @@ -25,7 +25,7 @@ public class QueryMatcher { public QueryMatcher() { for (SchemaElementType type : SchemaElementType.values()) { - if (type.equals(SchemaElementType.VIEW)) { + if (type.equals(SchemaElementType.DATASET)) { elementOptionMap.put(type, QueryMatchOption.optional()); } else { elementOptionMap.put(type, QueryMatchOption.unused()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java index 64159be43..907c85b21 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java @@ -102,9 +102,9 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { } private void fillSchemaElement(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) { - Set viewIds = parseInfo.getElementMatches().stream().map(SchemaElementMatch::getElement) - .map(SchemaElement::getView).collect(Collectors.toSet()); - parseInfo.setView(semanticSchema.getView(viewIds.iterator().next())); + Set dataSetIds = parseInfo.getElementMatches().stream().map(SchemaElementMatch::getElement) + .map(SchemaElement::getDataSet).collect(Collectors.toSet()); + parseInfo.setDataSet(semanticSchema.getDataSet(dataSetIds.iterator().next())); Map> dim2Values = new HashMap<>(); Map> id2Values = new HashMap<>(); @@ -189,7 +189,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public QueryResult execute(User user) { String queryMode = parseInfo.getQueryMode(); - if (parseInfo.getViewId() == null || StringUtils.isEmpty(queryMode) + if (parseInfo.getDataSetId() == null || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); @@ -230,7 +230,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public QueryResult multiStructExecute(User user) { String queryMode = parseInfo.getQueryMode(); - if (parseInfo.getViewId() != null || StringUtils.isEmpty(queryMode) + if (parseInfo.getDataSetId() != null || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java index 7496970a8..1983a8db9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.chat.core.query.rule.metric; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import org.springframework.stereotype.Component; + import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import org.springframework.stereotype.Component; @Component public class MetricModelQuery extends MetricSemanticQuery { @@ -14,7 +14,7 @@ public class MetricModelQuery extends MetricSemanticQuery { public MetricModelQuery() { super(); - queryMatcher.addOption(SchemaElementType.VIEW, OPTIONAL, AT_MOST, 1); + queryMatcher.addOption(SchemaElementType.DATASET, OPTIONAL, AT_MOST, 1); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java index c372663e3..ff66a1daf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java @@ -1,21 +1,22 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC; - +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import lombok.extern.slf4j.Slf4j; + import java.time.LocalDate; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; + +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC; @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { @@ -38,8 +39,9 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { super.fillParseInfo(queryContext, chatContext); parseInfo.setLimit(METRIC_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); - TimeDefaultConfig timeDefaultConfig = viewSchema.getMetricTypeTimeDefaultConfig(); + DataSetSchema dataSetSchema = + queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig(); DateConf dateInfo = new DateConf(); if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())) { int unit = timeDefaultConfig.getUnit(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java index c8a43df57..1e0e189b9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.query.rule.tag; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; @@ -25,19 +25,19 @@ public abstract class TagListQuery extends TagSemanticQuery { } private void addEntityDetailAndOrderByMetric(QueryContext queryContext, SemanticParseInfo parseInfo) { - Long viewId = parseInfo.getViewId(); - if (Objects.nonNull(viewId) && viewId > 0L) { - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); - if (viewSchema != null && Objects.nonNull(viewSchema.getEntity())) { + Long dataSetId = parseInfo.getDataSetId(); + if (Objects.nonNull(dataSetId) && dataSetId > 0L) { + DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + if (dataSetSchema != null && Objects.nonNull(dataSetSchema.getEntity())) { Set dimensions = new LinkedHashSet<>(); Set metrics = new LinkedHashSet<>(); Set orders = new LinkedHashSet<>(); - TagTypeDefaultConfig tagTypeDefaultConfig = viewSchema.getTagTypeDefaultConfig(); + TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig(); if (tagTypeDefaultConfig != null && tagTypeDefaultConfig.getDefaultDisplayInfo() != null) { if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds() .stream().map(id -> { - SchemaElement metric = viewSchema.getElement(SchemaElementType.METRIC, id); + SchemaElement metric = dataSetSchema.getElement(SchemaElementType.METRIC, id); if (metric != null) { orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)); } @@ -46,7 +46,7 @@ public abstract class TagListQuery extends TagSemanticQuery { } if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> viewSchema.getElement(SchemaElementType.DIMENSION, id)) + .map(id -> dataSetSchema.getElement(SchemaElementType.DIMENSION, id)) .filter(Objects::nonNull).collect(Collectors.toSet()); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java index 9abcd42e3..9f881a859 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.query.rule.tag; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; @@ -42,8 +42,9 @@ public abstract class TagSemanticQuery extends RuleSemanticQuery { parseInfo.setQueryType(QueryType.TAG); parseInfo.setLimit(TAG_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); - TimeDefaultConfig timeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + DataSetSchema dataSetSchema = + queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); DateConf dateInfo = new DateConf(); if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())) { int unit = timeDefaultConfig.getUnit(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java index 850499945..d34beedec 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.core.query.semantic; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -16,53 +16,53 @@ import java.util.concurrent.TimeUnit; @Slf4j public abstract class BaseSemanticInterpreter implements SemanticInterpreter { - protected final Cache> viewSchemaCache = + protected final Cache> dataSetSchemaCache = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); @SneakyThrows - public List fetchViewSchema(List ids, Boolean cacheEnable) { + public List fetchDataSetSchema(List ids, Boolean cacheEnable) { if (cacheEnable) { - return viewSchemaCache.get(String.valueOf(ids), () -> { - List data = doFetchViewSchema(ids); - viewSchemaCache.put(String.valueOf(ids), data); + return dataSetSchemaCache.get(String.valueOf(ids), () -> { + List data = doFetchDataSetSchema(ids); + dataSetSchemaCache.put(String.valueOf(ids), data); return data; }); } - return doFetchViewSchema(ids); + return doFetchDataSetSchema(ids); } @Override - public ViewSchema getViewSchema(Long viewId, Boolean cacheEnable) { + public DataSetSchema getDataSetSchema(Long dataSetId, Boolean cacheEnable) { List ids = new ArrayList<>(); - ids.add(viewId); - List viewSchemaResps = fetchViewSchema(ids, cacheEnable); - if (!CollectionUtils.isEmpty(viewSchemaResps)) { - Optional viewSchemaResp = viewSchemaResps.stream() - .filter(d -> d.getId().equals(viewId)).findFirst(); - if (viewSchemaResp.isPresent()) { - ViewSchemaResp viewSchema = viewSchemaResp.get(); - return ViewSchemaBuilder.build(viewSchema); + ids.add(dataSetId); + List dataSetSchemaResps = fetchDataSetSchema(ids, cacheEnable); + if (!CollectionUtils.isEmpty(dataSetSchemaResps)) { + Optional dataSetSchemaResp = dataSetSchemaResps.stream() + .filter(d -> d.getId().equals(dataSetId)).findFirst(); + if (dataSetSchemaResp.isPresent()) { + DataSetSchemaResp dataSetSchema = dataSetSchemaResp.get(); + return DataSetSchemaBuilder.build(dataSetSchema); } } return null; } @Override - public List getViewSchema() { - return getViewSchema(new ArrayList<>()); + public List getDataSetSchema() { + return getDataSetSchema(new ArrayList<>()); } @Override - public List getViewSchema(List ids) { - List domainSchemaList = new ArrayList<>(); + public List getDataSetSchema(List ids) { + List domainSchemaList = new ArrayList<>(); - for (ViewSchemaResp resp : fetchViewSchema(ids, true)) { - domainSchemaList.add(ViewSchemaBuilder.build(resp)); + for (DataSetSchemaResp resp : fetchDataSetSchema(ids, true)) { + domainSchemaList.add(DataSetSchemaBuilder.build(resp)); } return domainSchemaList; } - protected abstract List doFetchViewSchema(List ids); + protected abstract List doFetchDataSetSchema(List ids); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java index 1bbc151f6..b951cd00e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java @@ -5,13 +5,13 @@ import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; @@ -23,19 +23,19 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -public class ViewSchemaBuilder { +public class DataSetSchemaBuilder { - public static ViewSchema build(ViewSchemaResp resp) { - ViewSchema viewSchema = new ViewSchema(); - viewSchema.setQueryConfig(resp.getQueryConfig()); - SchemaElement model = SchemaElement.builder() - .view(resp.getId()) + public static DataSetSchema build(DataSetSchemaResp resp) { + DataSetSchema dataSetSchema = new DataSetSchema(); + dataSetSchema.setQueryConfig(resp.getQueryConfig()); + SchemaElement dataSet = SchemaElement.builder() + .dataSet(resp.getId()) .id(resp.getId()) .name(resp.getName()) .bizName(resp.getBizName()) - .type(SchemaElementType.VIEW) + .type(SchemaElementType.DATASET) .build(); - viewSchema.setView(model); + dataSetSchema.setDataSet(dataSet); Set metrics = new HashSet<>(); for (MetricSchemaResp metric : resp.getMetrics()) { @@ -43,7 +43,7 @@ public class ViewSchemaBuilder { List alias = SchemaItem.getAliasList(metric.getAlias()); SchemaElement metricToAdd = SchemaElement.builder() - .view(resp.getId()) + .dataSet(resp.getId()) .model(metric.getModelId()) .id(metric.getId()) .name(metric.getName()) @@ -57,7 +57,7 @@ public class ViewSchemaBuilder { metrics.add(metricToAdd); } - viewSchema.getMetrics().addAll(metrics); + dataSetSchema.getMetrics().addAll(metrics); Set dimensions = new HashSet<>(); Set dimensionValues = new HashSet<>(); @@ -84,7 +84,7 @@ public class ViewSchemaBuilder { } SchemaElement dimToAdd = SchemaElement.builder() - .view(resp.getId()) + .dataSet(resp.getId()) .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) @@ -97,7 +97,7 @@ public class ViewSchemaBuilder { dimensions.add(dimToAdd); SchemaElement dimValueToAdd = SchemaElement.builder() - .view(resp.getId()) + .dataSet(resp.getId()) .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) @@ -109,7 +109,7 @@ public class ViewSchemaBuilder { dimensionValues.add(dimValueToAdd); if (dim.getIsTag() == 1) { SchemaElement tagToAdd = SchemaElement.builder() - .view(resp.getId()) + .dataSet(resp.getId()) .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) @@ -122,14 +122,14 @@ public class ViewSchemaBuilder { tags.add(tagToAdd); } } - viewSchema.getDimensions().addAll(dimensions); - viewSchema.getDimensionValues().addAll(dimensionValues); - viewSchema.getTags().addAll(tags); + dataSetSchema.getDimensions().addAll(dimensions); + dataSetSchema.getDimensionValues().addAll(dimensionValues); + dataSetSchema.getTags().addAll(tags); DimSchemaResp dim = resp.getPrimaryKey(); if (dim != null) { SchemaElement entity = SchemaElement.builder() - .view(resp.getId()) + .dataSet(resp.getId()) .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) @@ -138,9 +138,9 @@ public class ViewSchemaBuilder { .useCnt(dim.getUseCnt()) .alias(dim.getEntityAlias()) .build(); - viewSchema.setEntity(entity); + dataSetSchema.setEntity(entity); } - return viewSchema; + return dataSetSchema; } private static List getRelateSchemaElement(MetricSchemaResp metricSchemaResp) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java index d8796f215..d7da709f0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java @@ -10,15 +10,15 @@ import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.QueryService; @@ -44,7 +44,7 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { QuerySqlReq querySqlReq = new QuerySqlReq(); querySqlReq.setSql(queryStructReq.getCorrectS2SQL()); - querySqlReq.setViewId(queryStructReq.getViewId()); + querySqlReq.setDataSetId(queryStructReq.getDataSetId()); querySqlReq.setParams(new ArrayList<>()); return queryByS2SQL(querySqlReq, user); } @@ -68,11 +68,11 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public List doFetchViewSchema(List ids) { - ViewFilterReq filter = new ViewFilterReq(); - filter.setViewIds(ids); + public List doFetchDataSetSchema(List ids) { + DataSetFilterReq filter = new DataSetFilterReq(); + filter.setDataSetIds(ids); schemaService = ContextUtils.getBean(SchemaService.class); - return schemaService.fetchViewSchema(filter); + return schemaService.fetchDataSetSchema(filter); } @Override @@ -82,9 +82,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public List getViewList(Long domainId) { + public List getDataSetList(Long domainId) { schemaService = ContextUtils.getBean(SchemaService.class); - return schemaService.getViewList(domainId); + return schemaService.getDataSetList(domainId); } @Override @@ -106,8 +106,8 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public List getDomainViewTree() { - return schemaService.getDomainViewTree(); + public List getDomainDataSetTree() { + return schemaService.getDomainDataSetTree(); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java index 9114640aa..39919f0f1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java @@ -24,8 +24,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -250,17 +250,17 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { } @Override - protected List doFetchViewSchema(List ids) { + protected List doFetchDataSetSchema(List ids) { return null; } @Override - public List getDomainViewTree() { + public List getDomainDataSetTree() { return null; } @Override - public List getViewList(Long domainId) { + public List getDataSetList(Long domainId) { return null; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java index 98662aedd..eaac77815 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.query.semantic; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; @@ -15,14 +15,14 @@ import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import java.util.List; /** - * A semantic layer provides a simplified and consistent view of data from multiple sources. - * It abstracts away the complexity of the underlying data sources and provides a unified view + * A semantic layer provides a simplified and consistent dataSet of data from multiple sources. + * It abstracts away the complexity of the underlying data sources and provides a unified dataSet * of the data that is easier to understand and use. *

* The interface defines methods for getting metadata as well as querying data in the semantic layer. @@ -39,11 +39,11 @@ public interface SemanticInterpreter { SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user); - List getViewSchema(); + List getDataSetSchema(); - List getViewSchema(List ids); + List getDataSetSchema(List ids); - ViewSchema getViewSchema(Long model, Boolean cacheEnable); + DataSetSchema getDataSetSchema(Long model, Boolean cacheEnable); PageInfo getDimensionPage(PageDimensionReq pageDimensionReq); @@ -53,10 +53,10 @@ public interface SemanticInterpreter { ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; - List fetchViewSchema(List ids, Boolean cacheEnable); + List fetchDataSetSchema(List ids, Boolean cacheEnable); - List getViewList(Long domainId); + List getDataSetList(Long domainId); - List getDomainViewTree(); + List getDomainDataSetTree(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java index cb80644ea..e3b3e842e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java @@ -1,22 +1,23 @@ package com.tencent.supersonic.chat.core.utils; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.parser.JavaLLMProxy; import com.tencent.supersonic.chat.core.parser.LLMProxy; -import com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver; +import com.tencent.supersonic.chat.core.parser.sql.llm.DataSetResolver; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.common.util.ContextUtils; -import java.util.Map; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.support.SpringFactoriesLoader; +import java.util.Map; +import java.util.Objects; + @Slf4j public class ComponentFactory { private static SemanticInterpreter semanticInterpreter; private static LLMProxy llmProxy; - private static ViewResolver modelResolver; + private static DataSetResolver modelResolver; public static SemanticInterpreter getSemanticLayer() { if (Objects.isNull(semanticInterpreter)) { @@ -44,9 +45,9 @@ public class ComponentFactory { return llmProxy; } - public static ViewResolver getModelResolver() { + public static DataSetResolver getModelResolver() { if (Objects.isNull(modelResolver)) { - modelResolver = init(ViewResolver.class); + modelResolver = init(DataSetResolver.class); } return modelResolver; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java index ca3ff8a40..0f1b04bd4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java @@ -37,8 +37,8 @@ public class QueryReqBuilder { public static QueryStructReq buildStructReq(SemanticParseInfo parseInfo) { QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setViewId(parseInfo.getViewId()); - queryStructReq.setViewName(parseInfo.getView().getName()); + queryStructReq.setDataSetId(parseInfo.getDataSetId()); + queryStructReq.setDataSetName(parseInfo.getDataSet().getName()); queryStructReq.setQueryType(parseInfo.getQueryType()); queryStructReq.setDateInfo(rewrite2Between(parseInfo.getDateInfo())); @@ -119,7 +119,7 @@ public class QueryReqBuilder { for (Filter dimensionFilter : queryStructReq.getDimensionFilters()) { QueryStructReq req = new QueryStructReq(); BeanUtils.copyProperties(queryStructReq, req); - req.setViewId(parseInfo.getViewId()); + req.setDataSetId(parseInfo.getDataSetId()); req.setDimensionFilters(Lists.newArrayList(dimensionFilter)); queryStructReqs.add(req); } @@ -131,15 +131,15 @@ public class QueryReqBuilder { * convert to QueryS2SQLReq * * @param querySql - * @param viewId + * @param dataSetId * @return */ - public static QuerySqlReq buildS2SQLReq(String querySql, Long viewId) { + public static QuerySqlReq buildS2SQLReq(String querySql, Long dataSetId) { QuerySqlReq querySQLReq = new QuerySqlReq(); if (Objects.nonNull(querySql)) { querySQLReq.setSql(querySql); } - querySQLReq.setViewId(viewId); + querySQLReq.setDataSetId(dataSetId); return querySQLReq; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java index 6da417634..f5762030b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java @@ -1,43 +1,44 @@ package com.tencent.supersonic.chat.core.utils; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.util.DatePeriodEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; -import java.util.Objects; import org.apache.commons.lang3.tuple.Pair; +import java.util.Objects; + public class S2SqlDateHelper { - public static String getReferenceDate(QueryContext queryContext, Long viewId) { + public static String getReferenceDate(QueryContext queryContext, Long dataSetId) { String defaultDate = DateUtils.getBeforeDate(0); - if (Objects.isNull(viewId)) { + if (Objects.isNull(dataSetId)) { return defaultDate; } - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); - if (viewSchema == null || viewSchema.getTagTypeTimeDefaultConfig() == null) { + DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + if (dataSetSchema == null || dataSetSchema.getTagTypeTimeDefaultConfig() == null) { return defaultDate; } - TimeDefaultConfig tagTypeTimeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + TimeDefaultConfig tagTypeTimeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); return getDefaultDate(defaultDate, tagTypeTimeDefaultConfig).getLeft(); } public static Pair getStartEndDate(QueryContext queryContext, - Long viewId, QueryType queryType) { + Long dataSetId, QueryType queryType) { String defaultDate = DateUtils.getBeforeDate(0); - if (Objects.isNull(viewId)) { + if (Objects.isNull(dataSetId)) { return Pair.of(defaultDate, defaultDate); } - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); - if (viewSchema == null) { + DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + if (dataSetSchema == null) { return Pair.of(defaultDate, defaultDate); } - TimeDefaultConfig defaultConfig = viewSchema.getMetricTypeTimeDefaultConfig(); + TimeDefaultConfig defaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig(); if (QueryType.TAG.equals(queryType)) { - defaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + defaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); } return getDefaultDate(defaultDate, defaultConfig); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java index 8eacf4246..892061ed3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java @@ -57,7 +57,7 @@ public class SimilarQueryManager { embeddingQuery.setQuery(queryText); Map metaData = new HashMap<>(); - metaData.put("modelId", similarQueryReq.getViewId()); + metaData.put("modelId", similarQueryReq.getDataSetId()); metaData.put("agentId", similarQueryReq.getAgentId()); embeddingQuery.setMetadata(metaData); String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java index 1abdfc6df..045323cf3 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult; import com.tencent.supersonic.chat.core.pojo.QueryContext; @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; class SchemaCorrectorTest { private String json = "{\n" - + " \"viewId\": 1,\n" + + " \"dataSetId\": 1,\n" + " \"llmReq\": {\n" + " \"queryText\": \"xxx2024年播放量最高的十首歌\",\n" + " \"filterCondition\": {\n" @@ -31,7 +31,7 @@ class SchemaCorrectorTest { + " },\n" + " \"schema\": {\n" + " \"domainName\": \"歌曲\",\n" - + " \"viewName\": \"歌曲\",\n" + + " \"dataSetName\": \"歌曲\",\n" + " \"fieldNameList\": [\n" + " \"商务组\",\n" + " \"歌曲名\",\n" @@ -52,7 +52,7 @@ class SchemaCorrectorTest { + " \"id\": \"y3LqVSRL\",\n" + " \"name\": \"大模型语义解析\",\n" + " \"type\": \"NL2SQL_LLM\",\n" - + " \"viewIds\": [\n" + + " \"dataSetIds\": [\n" + " 1\n" + " ]\n" + " },\n" @@ -63,8 +63,8 @@ class SchemaCorrectorTest { @Test void doCorrect() throws JsonProcessingException { - Long viewId = 1L; - QueryContext queryContext = buildQueryContext(viewId); + Long dataSetId = 1L; + QueryContext queryContext = buildQueryContext(dataSetId); ObjectMapper objectMapper = new ObjectMapper(); ParseResult parseResult = objectMapper.readValue(json, ParseResult.class); @@ -78,8 +78,8 @@ class SchemaCorrectorTest { semanticParseInfo.setSqlInfo(sqlInfo); SchemaElement schemaElement = new SchemaElement(); - schemaElement.setView(viewId); - semanticParseInfo.setView(schemaElement); + schemaElement.setDataSet(dataSetId); + semanticParseInfo.setDataSet(schemaElement); semanticParseInfo.getProperties().put(Constants.CONTEXT, parseResult); @@ -108,35 +108,35 @@ class SchemaCorrectorTest { } - private QueryContext buildQueryContext(Long viewId) { + private QueryContext buildQueryContext(Long dataSetId) { QueryContext queryContext = new QueryContext(); - List viewSchemaList = new ArrayList<>(); - ViewSchema viewSchema = new ViewSchema(); + List dataSetSchemaList = new ArrayList<>(); + DataSetSchema dataSetSchema = new DataSetSchema(); QueryConfig queryConfig = new QueryConfig(); - viewSchema.setQueryConfig(queryConfig); + dataSetSchema.setQueryConfig(queryConfig); SchemaElement schemaElement = new SchemaElement(); - schemaElement.setView(viewId); - viewSchema.setView(schemaElement); + schemaElement.setDataSet(dataSetId); + dataSetSchema.setDataSet(schemaElement); Set dimensions = new HashSet<>(); SchemaElement element1 = new SchemaElement(); - element1.setView(1L); + element1.setDataSet(1L); element1.setName("歌曲名"); dimensions.add(element1); SchemaElement element2 = new SchemaElement(); - element2.setView(1L); + element2.setDataSet(1L); element2.setName("商务组"); dimensions.add(element2); SchemaElement element3 = new SchemaElement(); - element3.setView(1L); + element3.setDataSet(1L); element3.setName("发行日期"); dimensions.add(element3); - viewSchema.setDimensions(dimensions); - viewSchemaList.add(viewSchema); + dataSetSchema.setDimensions(dimensions); + dataSetSchemaList.add(dataSetSchema); - SemanticSchema semanticSchema = new SemanticSchema(viewSchemaList); + SemanticSchema semanticSchema = new SemanticSchema(dataSetSchemaList); queryContext.setSemanticSchema(semanticSchema); return queryContext; } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java index 6f356ae1f..1782eae44 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java @@ -29,7 +29,7 @@ class LLMSqlParserTest { SchemaElement schemaElement = SchemaElement.builder() .bizName("singer_name") .name("歌手名") - .view(2L) + .dataSet(2L) .schemaValueMaps(schemaValueMaps) .build(); dimensions.add(schemaElement); @@ -37,7 +37,7 @@ class LLMSqlParserTest { SchemaElement schemaElement2 = SchemaElement.builder() .bizName("publish_time") .name("发布时间") - .view(2L) + .dataSet(2L) .build(); dimensions.add(schemaElement2); @@ -47,7 +47,7 @@ class LLMSqlParserTest { SchemaElement metric = SchemaElement.builder() .bizName("play_count") .name("播放量") - .view(2L) + .dataSet(2L) .build(); metrics.add(metric); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java index 54fd38e4b..b68392229 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.core.utils; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.QueryType; @@ -11,60 +11,60 @@ import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; -import java.util.ArrayList; -import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.junit.Assert; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; class S2SqlDateHelperTest { @Test void getReferenceDate() { - Long viewId = 1L; - QueryContext queryContext = buildQueryContext(viewId); + Long dataSetId = 1L; + QueryContext queryContext = buildQueryContext(dataSetId); String referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, null); Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(0)); - referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, dataSetId); Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(0)); - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); - QueryConfig queryConfig = viewSchema.getQueryConfig(); + DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + QueryConfig queryConfig = dataSetSchema.getQueryConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); timeDefaultConfig.setTimeMode(TimeMode.LAST); timeDefaultConfig.setPeriod(Constants.DAY); timeDefaultConfig.setUnit(20); queryConfig.getTagTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); - referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, dataSetId); Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(20)); timeDefaultConfig.setUnit(1); - referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, dataSetId); Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(1)); timeDefaultConfig.setUnit(-1); - referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, dataSetId); Assert.assertNull(referenceDate); } @Test void getStartEndDate() { - Long viewId = 1L; - QueryContext queryContext = buildQueryContext(viewId); + Long dataSetId = 1L; + QueryContext queryContext = buildQueryContext(dataSetId); Pair startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, null, QueryType.TAG); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(0)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(0)); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.TAG); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.TAG); Assert.assertNull(startEndDate.getLeft()); Assert.assertNull(startEndDate.getRight()); - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); - QueryConfig queryConfig = viewSchema.getQueryConfig(); + DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); + QueryConfig queryConfig = dataSetSchema.getQueryConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); timeDefaultConfig.setTimeMode(TimeMode.LAST); timeDefaultConfig.setPeriod(Constants.DAY); @@ -72,49 +72,49 @@ class S2SqlDateHelperTest { queryConfig.getTagTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); queryConfig.getMetricTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.TAG); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.TAG); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(20)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(20)); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.METRIC); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(20)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(20)); timeDefaultConfig.setUnit(2); timeDefaultConfig.setTimeMode(TimeMode.RECENT); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.METRIC); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(2)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(1)); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.TAG); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.TAG); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(2)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(1)); timeDefaultConfig.setUnit(-1); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.METRIC); Assert.assertNull(startEndDate.getLeft()); Assert.assertNull(startEndDate.getRight()); timeDefaultConfig.setTimeMode(TimeMode.LAST); timeDefaultConfig.setPeriod(Constants.DAY); timeDefaultConfig.setUnit(5); - startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, dataSetId, QueryType.METRIC); Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(5)); Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(5)); } - private QueryContext buildQueryContext(Long viewId) { + private QueryContext buildQueryContext(Long dataSetId) { QueryContext queryContext = new QueryContext(); - List viewSchemaList = new ArrayList<>(); - ViewSchema viewSchema = new ViewSchema(); + List dataSetSchemaList = new ArrayList<>(); + DataSetSchema dataSetSchema = new DataSetSchema(); QueryConfig queryConfig = new QueryConfig(); - viewSchema.setQueryConfig(queryConfig); + dataSetSchema.setQueryConfig(queryConfig); SchemaElement schemaElement = new SchemaElement(); - schemaElement.setView(viewId); - viewSchema.setView(schemaElement); - viewSchemaList.add(viewSchema); + schemaElement.setDataSet(dataSetId); + dataSetSchema.setDataSet(schemaElement); + dataSetSchemaList.add(dataSetSchema); - SemanticSchema semanticSchema = new SemanticSchema(viewSchemaList); + SemanticSchema semanticSchema = new SemanticSchema(dataSetSchemaList); queryContext.setSemanticSchema(semanticSchema); return queryContext; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java index 3d6a18d22..c4efb4744 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java @@ -15,7 +15,7 @@ public class PluginDO { private String type; - private String view; + private String dataSet; private String pattern; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index 5c599c81e..d1e312621 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -4,7 +4,7 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.service.SemanticService; @@ -34,15 +34,15 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { return; } SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); - List dimensionRecommended = getDimensions(element.getId(), element.getView()); + List dimensionRecommended = getDimensions(element.getId(), element.getDataSet()); queryResult.setRecommendedDimensions(dimensionRecommended); } - private List getDimensions(Long metricId, Long viewId) { + private List getDimensions(Long metricId, Long dataSetId) { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ViewSchema viewSchema = semanticService.getViewSchema(viewId); + DataSetSchema dataSetSchema = semanticService.getDataSetSchema(dataSetId); List drillDownDimensions = Lists.newArrayList(); - Set metricElements = viewSchema.getMetrics(); + Set metricElements = dataSetSchema.getMetrics(); if (!CollectionUtils.isEmpty(metricElements)) { Optional metric = metricElements.stream().filter(schemaElement -> metricId.equals(schemaElement.getId()) @@ -54,7 +54,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { } } final List drillDownDimensionsFinal = drillDownDimensions; - return viewSchema.getDimensions().stream() + return dataSetSchema.getDimensions().stream() .filter(dim -> filterDimension(drillDownDimensionsFinal, dim)) .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(recommend_dimension_size) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java index 7f3330ec6..ec91b4855 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.alibaba.fastjson.JSONObject; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.common.config.EmbeddingConfig; +import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ComponentFactory; import com.tencent.supersonic.common.util.ContextUtils; @@ -14,6 +13,9 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.server.service.MetaEmbeddingService; import org.springframework.util.CollectionUtils; import java.util.Collections; @@ -46,15 +48,14 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { } List metricNames = Collections.singletonList(parseInfo.getMetrics().iterator().next().getName()); Map filterCondition = new HashMap<>(); - filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getView().toString()); + filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getDataSet().toString()); filterCondition.put("type", SchemaElementType.METRIC.name()); RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames) .filterCondition(filterCondition).queryEmbeddings(null).build(); - - EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); - - List retrieveQueryResults = s2EmbeddingStore.retrieveQuery( - embeddingConfig.getMetaCollectionName(), retrieveQuery, METRIC_RECOMMEND_SIZE + 1); + MetaEmbeddingService metaEmbeddingService = ContextUtils.getBean(MetaEmbeddingService.class); + List retrieveQueryResults = + metaEmbeddingService.retrieveQuery(Lists.newArrayList(parseInfo.getDataSetId()), + retrieveQuery, METRIC_RECOMMEND_SIZE + 1); if (CollectionUtils.isEmpty(retrieveQueryResults)) { return; } @@ -71,9 +72,10 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { if (!metricIds.contains(Retrieval.getLongId(retrieval.getId()))) { SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(retrieval.getMetadata()), SchemaElement.class); - if (retrieval.getMetadata().containsKey("viewId")) { - String viewId = retrieval.getMetadata().get("viewId").toString(); - schemaElement.setView(Long.parseLong(viewId)); + if (retrieval.getMetadata().containsKey("dataSetId")) { + String dataSetId = retrieval.getMetadata().get("dataSetId").toString() + .replace(Constants.UNDERLINE, ""); + schemaElement.setDataSet(Long.parseLong(dataSetId)); } schemaElement.setOrder(++metricOrder); parseInfo.getMetrics().add(schemaElement); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java index 30e6c4eb0..dbac4bf64 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.processor.parse; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.core.pojo.ChatContext; @@ -37,9 +37,10 @@ public class EntityInfoProcessor implements ParseResultProcessor { return; } //1. set entity info - ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); + DataSetSchema dataSetSchema = + queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, viewSchema, queryContext.getUser()); + EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, queryContext.getUser()); if (QueryManager.isTagQuery(queryMode) || QueryManager.isMetricQuery(queryMode)) { parseInfo.setEntityInfo(entityInfo); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java index b844974f7..979fa8ec1 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java @@ -74,9 +74,9 @@ public class ParseInfoProcessor implements ParseResultProcessor { } //set filter - Long viewId = parseInfo.getViewId(); + Long dataSetId = parseInfo.getDataSetId(); try { - Map fieldNameToElement = getNameToElement(viewId); + Map fieldNameToElement = getNameToElement(dataSetId); List result = getDimensionFilter(fieldNameToElement, expressions); parseInfo.getDimensionFilters().addAll(result); } catch (Exception e) { @@ -88,31 +88,31 @@ public class ParseInfoProcessor implements ParseResultProcessor { return; } List allFields = getFieldsExceptDate(SqlSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL())); - Set metrics = getElements(viewId, allFields, semanticSchema.getMetrics()); + Set metrics = getElements(dataSetId, allFields, semanticSchema.getMetrics()); parseInfo.setMetrics(metrics); if (QueryType.METRIC.equals(parseInfo.getQueryType())) { List groupByFields = SqlSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); List groupByDimensions = getFieldsExceptDate(groupByFields); - parseInfo.setDimensions(getElements(viewId, groupByDimensions, semanticSchema.getDimensions())); + parseInfo.setDimensions(getElements(dataSetId, groupByDimensions, semanticSchema.getDimensions())); } else if (QueryType.TAG.equals(parseInfo.getQueryType())) { List selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); List selectDimensions = getFieldsExceptDate(selectFields); - parseInfo.setDimensions(getElements(viewId, selectDimensions, semanticSchema.getDimensions())); + parseInfo.setDimensions(getElements(dataSetId, selectDimensions, semanticSchema.getDimensions())); } } - private Set getElements(Long viewId, List allFields, List elements) { + private Set getElements(Long dataSetId, List allFields, List elements) { return elements.stream() .filter(schemaElement -> { if (CollectionUtils.isEmpty(schemaElement.getAlias())) { - return viewId.equals(schemaElement.getView()) && allFields.contains( + return dataSetId.equals(schemaElement.getDataSet()) && allFields.contains( schemaElement.getName()); } Set allFieldsSet = new HashSet<>(allFields); Set aliasSet = new HashSet<>(schemaElement.getAlias()); List intersection = allFieldsSet.stream() .filter(aliasSet::contains).collect(Collectors.toList()); - return viewId.equals(schemaElement.getView()) && (allFields.contains( + return dataSetId.equals(schemaElement.getDataSet()) && (allFields.contains( schemaElement.getName()) || !CollectionUtils.isEmpty(intersection)); } ).collect(Collectors.toSet()); @@ -194,10 +194,10 @@ public class ParseInfoProcessor implements ParseResultProcessor { return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue()); } - protected Map getNameToElement(Long viewId) { + protected Map getNameToElement(Long dataSetId) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - List dimensions = semanticSchema.getDimensions(viewId); - List metrics = semanticSchema.getMetrics(viewId); + List dimensions = semanticSchema.getDimensions(dataSetId); + List metrics = semanticSchema.getMetrics(dataSetId); List allElements = Lists.newArrayList(); allElements.addAll(dimensions); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java index ebada89d4..01d6f8a43 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -84,14 +84,14 @@ public class ChatConfigController { } //Compatible with front-end - @GetMapping("/viewList") - public List getViewList() { - return semanticInterpreter.getViewList(null); + @GetMapping("/dataSetList") + public List getDataSetList() { + return semanticInterpreter.getDataSetList(null); } - @GetMapping("/viewList/{domainId}") - public List getViewList(@PathVariable("domainId") Long domainId) { - return semanticInterpreter.getViewList(domainId); + @GetMapping("/dataSetList/{domainId}") + public List getDataSetList(@PathVariable("domainId") Long domainId) { + return semanticInterpreter.getDataSetList(domainId); } @PostMapping("/dimension/page") @@ -107,9 +107,9 @@ public class ChatConfigController { return semanticInterpreter.getMetricPage(pageMetricReq, user); } - @GetMapping("/getDomainViewTree") - public List getDomainViewTree() { - return semanticInterpreter.getDomainViewTree(); + @GetMapping("/getDomainDataSetTree") + public List getDomainDataSetTree() { + return semanticInterpreter.getDomainDataSetTree(); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java index ec9423da2..f6d79a51d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java @@ -6,11 +6,11 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.DataInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; -import com.tencent.supersonic.chat.api.pojo.response.ViewInfo; +import com.tencent.supersonic.chat.api.pojo.response.DataSetInfo; import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; @@ -50,19 +50,19 @@ public class SemanticService { return schemaService.getSemanticSchema(); } - public ViewSchema getViewSchema(Long id) { - return schemaService.getViewSchema(id); + public DataSetSchema getDataSetSchema(Long id) { + return schemaService.getDataSetSchema(id); } - public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, ViewSchema viewSchema, User user) { - if (parseInfo != null && parseInfo.getViewId() > 0) { - EntityInfo entityInfo = getEntityBasicInfo(viewSchema); - if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getViewInfo() == null) { + public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) { + if (parseInfo != null && parseInfo.getDataSetId() > 0) { + EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema); + if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) { entityInfo.setMetrics(null); entityInfo.setDimensions(null); return entityInfo; } - String primaryKey = entityInfo.getViewInfo().getPrimaryKey(); + String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey(); if (StringUtils.isNotBlank(primaryKey)) { String entityId = ""; for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { @@ -75,7 +75,7 @@ public class SemanticService { } entityInfo.setEntityId(entityId); try { - fillEntityInfoValue(entityInfo, viewSchema, user); + fillEntityInfoValue(entityInfo, dataSetSchema, user); return entityInfo; } catch (Exception e) { log.error("setMainModel error", e); @@ -85,29 +85,29 @@ public class SemanticService { return null; } - private EntityInfo getEntityBasicInfo(ViewSchema viewSchema) { + private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) { EntityInfo entityInfo = new EntityInfo(); - if (viewSchema == null) { + if (dataSetSchema == null) { return entityInfo; } - Long viewId = viewSchema.getView().getView(); - ViewInfo viewInfo = new ViewInfo(); - viewInfo.setItemId(viewId.intValue()); - viewInfo.setName(viewSchema.getView().getName()); - viewInfo.setWords(viewSchema.getView().getAlias()); - viewInfo.setBizName(viewSchema.getView().getBizName()); - if (Objects.nonNull(viewSchema.getEntity())) { - viewInfo.setPrimaryKey(viewSchema.getEntity().getBizName()); + Long dataSetId = dataSetSchema.getDataSet().getDataSet(); + DataSetInfo dataSetInfo = new DataSetInfo(); + dataSetInfo.setItemId(dataSetId.intValue()); + dataSetInfo.setName(dataSetSchema.getDataSet().getName()); + dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias()); + dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName()); + if (Objects.nonNull(dataSetSchema.getEntity())) { + dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName()); } - entityInfo.setViewInfo(viewInfo); - TagTypeDefaultConfig tagTypeDefaultConfig = viewSchema.getTagTypeDefaultConfig(); + entityInfo.setDataSetInfo(dataSetInfo); + TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig(); if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) { return entityInfo; } List dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() .map(id -> { - SchemaElement element = viewSchema.getElement(SchemaElementType.DIMENSION, id); + SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id); if (element == null) { return null; } @@ -115,7 +115,7 @@ public class SemanticService { }).filter(Objects::nonNull).collect(Collectors.toList()); List metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() .map(id -> { - SchemaElement element = viewSchema.getElement(SchemaElementType.METRIC, id); + SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id); if (element == null) { return null; } @@ -126,9 +126,9 @@ public class SemanticService { return entityInfo; } - public void fillEntityInfoValue(EntityInfo entityInfo, ViewSchema viewSchema, User user) { + public void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) { SemanticQueryResp queryResultWithColumns = - getQueryResultWithSchemaResp(entityInfo, viewSchema, user); + getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user); if (queryResultWithColumns != null) { if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList()) && queryResultWithColumns.getResultList().size() > 0) { @@ -147,15 +147,16 @@ public class SemanticService { } } - public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, ViewSchema viewSchema, User user) { + public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, + DataSetSchema dataSetSchema, User user) { SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - semanticParseInfo.setView(viewSchema.getView()); + semanticParseInfo.setDataSet(dataSetSchema.getDataSet()); semanticParseInfo.setQueryType(QueryType.TAG); semanticParseInfo.setMetrics(getMetrics(entityInfo)); semanticParseInfo.setDimensions(getDimensions(entityInfo)); DateConf dateInfo = new DateConf(); int unit = 1; - TimeDefaultConfig timeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); if (Objects.nonNull(timeDefaultConfig)) { unit = timeDefaultConfig.getUnit(); String date = LocalDate.now().plusDays(-unit).toString(); @@ -222,7 +223,7 @@ public class SemanticService { } private String getEntityPrimaryName(EntityInfo entityInfo) { - return entityInfo.getViewInfo().getPrimaryKey(); + return entityInfo.getDataSetInfo().getPrimaryKey(); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java index 2b8f365b9..ce6c8f222 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java @@ -60,8 +60,8 @@ public class ChatServiceImpl implements ChatService { return null; } SemanticParseInfo originalSemanticParse = chatContext.getParseInfo(); - if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getViewId())) { - return originalSemanticParse.getViewId(); + if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getDataSetId())) { + return originalSemanticParse.getDataSetId(); } return null; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index d3b6b4cc2..9f2522a81 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -177,7 +177,7 @@ public class ConfigServiceImpl implements ConfigService { } private ItemVisibilityInfo fetchVisibilityDescByConfig(ItemVisibility visibility, - ViewSchema modelSchema) { + DataSetSchema modelSchema) { ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo(); List dimIdAllList = chatConfigHelper.generateAllDimIdList(modelSchema); @@ -219,20 +219,20 @@ public class ConfigServiceImpl implements ConfigService { } BeanUtils.copyProperties(chatConfigResp, chatConfigRich); - ViewSchema viewSchema = semanticService.getViewSchema(modelId); - if (viewSchema == null) { + DataSetSchema dataSetSchema = semanticService.getDataSetSchema(modelId); + if (dataSetSchema == null) { return chatConfigRich; } - chatConfigRich.setBizName(viewSchema.getView().getBizName()); - chatConfigRich.setModelName(viewSchema.getView().getName()); + chatConfigRich.setBizName(dataSetSchema.getDataSet().getBizName()); + chatConfigRich.setModelName(dataSetSchema.getDataSet().getName()); - chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(viewSchema, chatConfigResp)); - chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(viewSchema, chatConfigRich, chatConfigResp)); + chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(dataSetSchema, chatConfigResp)); + chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(dataSetSchema, chatConfigRich, chatConfigResp)); return chatConfigRich; } - private ChatDetailRichConfigResp fillChatDetailRichConfig(ViewSchema modelSchema, + private ChatDetailRichConfigResp fillChatDetailRichConfig(DataSetSchema modelSchema, ChatConfigRichResp chatConfigRich, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) { @@ -251,7 +251,7 @@ public class ConfigServiceImpl implements ConfigService { return detailRichConfig; } - private EntityRichInfoResp generateRichEntity(Entity entity, ViewSchema modelSchema) { + private EntityRichInfoResp generateRichEntity(Entity entity, DataSetSchema modelSchema) { EntityRichInfoResp entityRichInfo = new EntityRichInfoResp(); if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) { return entityRichInfo; @@ -264,7 +264,7 @@ public class ConfigServiceImpl implements ConfigService { return entityRichInfo; } - private ChatAggRichConfigResp fillChatAggRichConfig(ViewSchema modelSchema, ChatConfigResp chatConfigResp) { + private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) { return null; } @@ -281,7 +281,7 @@ public class ConfigServiceImpl implements ConfigService { } private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig, - ViewSchema modelSchema, + DataSetSchema modelSchema, ItemVisibilityInfo itemVisibilityInfo) { ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp(); if (Objects.isNull(chatDefaultConfig)) { @@ -331,7 +331,7 @@ public class ConfigServiceImpl implements ConfigService { } private List fillKnowledgeBizName(List knowledgeInfos, - ViewSchema modelSchema) { + DataSetSchema modelSchema) { if (CollectionUtils.isEmpty(knowledgeInfos)) { return new ArrayList<>(); } @@ -351,9 +351,9 @@ public class ConfigServiceImpl implements ConfigService { @Override public List getAllChatRichConfig() { List chatConfigRichInfoList = new ArrayList<>(); - List modelSchemas = semanticInterpreter.getViewSchema(); + List modelSchemas = semanticInterpreter.getDataSetSchema(); modelSchemas.stream().forEach(modelSchema -> { - ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getView().getId()); + ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getDataSet().getId()); if (Objects.nonNull(chatConfigRichInfo)) { chatConfigRichInfoList.add(chatConfigRichInfo); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java index f0dff6895..6353004a2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java @@ -93,8 +93,8 @@ public class PluginServiceImpl implements PluginService { if (StringUtils.isNotBlank(pluginQueryReq.getType())) { queryWrapper.lambda().eq(PluginDO::getType, pluginQueryReq.getType()); } - if (StringUtils.isNotBlank(pluginQueryReq.getView())) { - queryWrapper.lambda().like(PluginDO::getView, pluginQueryReq.getView()); + if (StringUtils.isNotBlank(pluginQueryReq.getDataSet())) { + queryWrapper.lambda().like(PluginDO::getDataSet, pluginQueryReq.getDataSet()); } if (StringUtils.isNotBlank(pluginQueryReq.getParseMode())) { queryWrapper.lambda().eq(PluginDO::getParseMode, pluginQueryReq.getParseMode()); @@ -180,8 +180,8 @@ public class PluginServiceImpl implements PluginService { public Plugin convert(PluginDO pluginDO) { Plugin plugin = new Plugin(); BeanUtils.copyProperties(pluginDO, plugin); - if (StringUtils.isNotBlank(pluginDO.getView())) { - plugin.setViewList(Arrays.stream(pluginDO.getView().split(",")) + if (StringUtils.isNotBlank(pluginDO.getDataSet())) { + plugin.setDataSetList(Arrays.stream(pluginDO.getDataSet().split(",")) .map(Long::parseLong).collect(Collectors.toList())); } return plugin; @@ -194,7 +194,7 @@ public class PluginServiceImpl implements PluginService { pluginDO.setCreatedBy(user.getName()); pluginDO.setUpdatedAt(new Date()); pluginDO.setUpdatedBy(user.getName()); - pluginDO.setView(StringUtils.join(plugin.getViewList(), ",")); + pluginDO.setDataSet(StringUtils.join(plugin.getDataSetList(), ",")); return pluginDO; } @@ -202,7 +202,7 @@ public class PluginServiceImpl implements PluginService { BeanUtils.copyProperties(plugin, pluginDO); pluginDO.setUpdatedAt(new Date()); pluginDO.setUpdatedBy(user.getName()); - pluginDO.setView(StringUtils.join(plugin.getViewList(), ",")); + pluginDO.setDataSet(StringUtils.join(plugin.getDataSetList(), ",")); return pluginDO; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java index 469b983a0..5ea5e73bb 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.chat.server.service.impl; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; @@ -296,7 +295,7 @@ public class QueryServiceImpl implements QueryService { similarQueryManager.saveSimilarQuery(SimilarQueryReq.builder().parseId(queryReq.getParseId()) .queryId(queryReq.getQueryId()) .agentId(chatQueryDO.getAgentId()) - .viewId(parseInfo.getViewId()) + .dataSetId(parseInfo.getDataSetId()) .queryText(queryReq.getQueryText()).build()); } @@ -352,9 +351,9 @@ public class QueryServiceImpl implements QueryService { } QueryResult queryResult = semanticQuery.execute(user); queryResult.setChatContext(semanticQuery.getParseInfo()); - ViewSchema viewSchema = semanticSchema.getViewSchemaMap().get(parseInfo.getViewId()); + DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(parseInfo.getDataSetId()); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, viewSchema, user); + EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user); queryResult.setEntityInfo(entityInfo); return queryResult; } @@ -422,8 +421,9 @@ public class QueryServiceImpl implements QueryService { ChatParseDO chatParseDO = chatService.getParseInfo(queryId, parseId); SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ViewSchema viewSchema = schemaService.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); - return semanticService.getEntityInfo(parseInfo, viewSchema, user); + DataSetSchema dataSetSchema = + schemaService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + return semanticService.getEntityInfo(parseInfo, dataSetSchema, user); } private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo, @@ -641,10 +641,10 @@ public class QueryServiceImpl implements QueryService { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); SemanticSchema semanticSchema = semanticService.getSemanticSchema(); SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID()); - Set detectViewIds = new HashSet<>(); - detectViewIds.add(schemaElement.getView()); - dimensionValueReq.setModelId(schemaElement.getView()); - List dimensionValues = getDimensionValues(dimensionValueReq, detectViewIds); + Set detectDataSetIds = new HashSet<>(); + detectDataSetIds.add(schemaElement.getDataSet()); + dimensionValueReq.setModelId(schemaElement.getDataSet()); + List dimensionValues = getDimensionValues(dimensionValueReq, detectDataSetIds); // if the search results is null,search dimensionValue from database if (CollectionUtils.isEmpty(dimensionValues)) { semanticQueryResp = queryDatabase(dimensionValueReq, user); @@ -668,14 +668,14 @@ public class QueryServiceImpl implements QueryService { return semanticQueryResp; } - private List getDimensionValues(DimensionValueReq dimensionValueReq, Set viewIds) { + private List getDimensionValues(DimensionValueReq dimensionValueReq, Set dataSetIds) { //if value is null ,then search from NATURE_TO_VALUES if (StringUtils.isBlank(dimensionValueReq.getValue())) { return SearchService.getDimensionValue(dimensionValueReq); } //search from prefixSearch List hanlpMapResultList = knowledgeService.prefixSearch(dimensionValueReq.getValue(), - 2000, viewIds); + 2000, dataSetIds); HanlpHelper.transLetterOriginal(hanlpMapResultList); return hanlpMapResultList.stream() .filter(o -> { @@ -700,7 +700,7 @@ public class QueryServiceImpl implements QueryService { dateConf.setPeriod("DAY"); queryStructReq.setDateInfo(dateConf); queryStructReq.setLimit(20L); - queryStructReq.setViewId(dimensionValueReq.getModelId()); + queryStructReq.setDataSetId(dimensionValueReq.getModelId()); queryStructReq.setQueryType(QueryType.ID); List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java index 9d76b7bca..b8d79388d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.RecommendReq; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; @@ -48,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService { if (Objects.isNull(modelId)) { return new RecommendResp(); } - ViewSchema modelSchema = semanticService.getViewSchema(modelId); + DataSetSchema modelSchema = semanticService.getDataSetSchema(modelId); if (Objects.isNull(modelSchema)) { return new RecommendResp(); } @@ -80,7 +80,7 @@ public class RecommendServiceImpl implements RecommendService { .limit(limit) .map(dimSchemaDesc -> { SchemaElement item = new SchemaElement(); - item.setView(modelId); + item.setDataSet(modelId); item.setName(dimSchemaDesc.getName()); item.setBizName(dimSchemaDesc.getBizName()); item.setId(dimSchemaDesc.getId()); @@ -94,7 +94,7 @@ public class RecommendServiceImpl implements RecommendService { .limit(limit) .map(metricSchemaDesc -> { SchemaElement item = new SchemaElement(); - item.setView(modelId); + item.setDataSet(modelId); item.setName(metricSchemaDesc.getName()); item.setBizName(metricSchemaDesc.getBizName()); item.setId(metricSchemaDesc.getId()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java index 19ba854d1..0fca0abe6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java @@ -4,7 +4,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; @@ -28,13 +28,13 @@ public class SchemaService { @Override public SemanticSchema load(String key) { log.info("load getDomainSchemaInfo cache [{}]", key); - return new SemanticSchema(semanticInterpreter.getViewSchema()); + return new SemanticSchema(semanticInterpreter.getDataSetSchema()); } } ); - public ViewSchema getViewSchema(Long id) { - return semanticInterpreter.getViewSchema(id, true); + public DataSetSchema getDataSetSchema(Long id) { + return semanticInterpreter.getDataSetSchema(id, true); } public SemanticSchema getSemanticSchema() { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java index 5518ab7be..6d2f7eacb 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java @@ -14,7 +14,7 @@ import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat; +import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat; import com.tencent.supersonic.chat.core.mapper.MapperHelper; import com.tencent.supersonic.chat.core.mapper.MatchText; import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType; @@ -91,18 +91,19 @@ public class SearchServiceImpl implements SearchService { // 2.get meta info SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema(); List metricsDb = semanticSchemaDb.getMetrics(); - final Map modelToName = semanticSchemaDb.getViewIdToName(); + final Map modelToName = semanticSchemaDb.getDataSetIdToName(); // 3.detect by segment List originals = knowledgeService.getTerms(queryText); log.info("hanlp parse result: {}", originals); MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); - Set detectViewIds = mapperHelper.getViewIds(queryReq.getViewId(), agentService.getAgent(agentId)); + Set detectDataSetIds = mapperHelper.getDataSetIds(queryReq.getDataSetId(), + agentService.getAgent(agentId)); QueryContext queryContext = new QueryContext(); BeanUtils.copyProperties(queryReq, queryContext); Map> regTextMap = - searchMatchStrategy.match(queryContext, originals, detectViewIds); + searchMatchStrategy.match(queryContext, originals, detectDataSetIds); regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue())); // 4.get the most matching data @@ -121,9 +122,9 @@ public class SearchServiceImpl implements SearchService { log.info("searchTextEntry:{},queryReq:{}", searchTextEntry, queryReq); Set searchResults = new LinkedHashSet(); - ViewInfoStat modelStat = NatureHelper.getViewStat(originals); + DataSetInfoStat modelStat = NatureHelper.getDataSetStat(originals); - List possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getViewId()); + List possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getDataSetId()); // 5.1 priority dimension metric boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleModels), modelToName, @@ -137,7 +138,7 @@ public class SearchServiceImpl implements SearchService { for (Map.Entry natureToNameEntry : natureToNameMap.entrySet()) { Set searchResultSet = searchDimensionValue(metricsDb, modelToName, - modelStat.getMetricViewCount(), existMetricAndDimension, + modelStat.getMetricDataSetCount(), existMetricAndDimension, matchText, natureToNameMap, natureToNameEntry, queryReq.getQueryFilters()); searchResults.addAll(searchResultSet); @@ -146,7 +147,7 @@ public class SearchServiceImpl implements SearchService { } private List getPossibleModels(QueryReq queryCtx, List originals, - ViewInfoStat modelStat, Long webModelId) { + DataSetInfoStat modelStat, Long webModelId) { if (Objects.nonNull(webModelId) && webModelId > 0) { List result = new ArrayList<>(); @@ -154,7 +155,7 @@ public class SearchServiceImpl implements SearchService { return result; } - List possibleModels = NatureHelper.selectPossibleViews(originals); + List possibleModels = NatureHelper.selectPossibleDataSets(originals); Long contextModel = chatService.getContextModel(queryCtx.getChatId()); @@ -167,9 +168,9 @@ public class SearchServiceImpl implements SearchService { return possibleModels; } - private boolean nothingOrOnlyMetric(ViewInfoStat modelStat) { - return modelStat.getMetricViewCount() >= 0 && modelStat.getDimensionViewCount() <= 0 - && modelStat.getDimensionValueViewCount() <= 0 && modelStat.getViewCount() <= 0; + private boolean nothingOrOnlyMetric(DataSetInfoStat modelStat) { + return modelStat.getMetricDataSetCount() >= 0 && modelStat.getDimensionDataSetCount() <= 0 + && modelStat.getDimensionValueDataSetCount() <= 0 && modelStat.getDataSetCount() <= 0; } private boolean effectiveModel(Long contextModel) { @@ -189,7 +190,7 @@ public class SearchServiceImpl implements SearchService { String nature = natureToNameEntry.getKey(); String wordName = natureToNameEntry.getValue(); - Long modelId = NatureHelper.getViewId(nature); + Long modelId = NatureHelper.getDataSetId(nature); SchemaElementType schemaElementType = NatureHelper.convertToElementType(nature); if (SchemaElementType.ENTITY.equals(schemaElementType)) { @@ -266,7 +267,7 @@ public class SearchServiceImpl implements SearchService { return Lists.newArrayList(); } return metricsDb.stream() - .filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getView())) + .filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getDataSet())) .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .flatMap(entry -> { List result = new ArrayList<>(); @@ -290,7 +291,7 @@ public class SearchServiceImpl implements SearchService { if (CollectionUtils.isEmpty(possibleModels)) { return true; } - Long model = NatureHelper.getViewId(nature); + Long model = NatureHelper.getDataSetId(nature); return possibleModels.contains(model); }) .map(nature -> { @@ -313,7 +314,7 @@ public class SearchServiceImpl implements SearchService { for (HanlpMapResult hanlpMapResult : hanlpMapResults) { List dimensionMetricClassIds = hanlpMapResult.getNatures().stream() - .map(nature -> new ModelWithSemanticType(NatureHelper.getViewId(nature), + .map(nature -> new ModelWithSemanticType(NatureHelper.getDataSetId(nature), NatureHelper.convertToElementType(nature))) .filter(entry -> matchCondition(entry, possibleModels)).collect(Collectors.toList()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java index 14e2c5b90..45adacd00 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java @@ -24,7 +24,7 @@ public class WordService { public List getAllDictWords() { SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getViewSchema()); + SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getDataSetSchema()); List words = new ArrayList<>(); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java index e47b84786..a460c7b6f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.util; import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -56,7 +56,7 @@ public class ChatConfigHelper { return chatConfig; } - public List generateAllDimIdList(ViewSchema modelSchema) { + public List generateAllDimIdList(DataSetSchema modelSchema) { if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getDimensions())) { return new ArrayList<>(); } @@ -65,7 +65,7 @@ public class ChatConfigHelper { return new ArrayList<>(dimIdAndDescPair.keySet()); } - public List generateAllMetricIdList(ViewSchema modelSchema) { + public List generateAllMetricIdList(DataSetSchema modelSchema) { if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getMetrics())) { return new ArrayList<>(); } diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java index 59e3dc51d..6e3e7c66a 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java @@ -30,8 +30,8 @@ class QueryReqBuilderTest { void buildS2SQLReq() { init(); QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setViewId(1L); - queryStructReq.setViewName("内容库"); + queryStructReq.setDataSetId(1L); + queryStructReq.setDataSetName("内容库"); queryStructReq.setQueryType(QueryType.METRIC); Aggregator aggregator = new Aggregator(); diff --git a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java index fc2850513..745555398 100644 --- a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java +++ b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java @@ -29,7 +29,7 @@ public class LoadRemoveService { if (Objects.isNull(nature)) { return false; } - Long modelId = getViewId(nature); + Long modelId = getDataSetId(nature); if (Objects.nonNull(modelId)) { return !detectModelIds.contains(modelId); } @@ -47,7 +47,7 @@ public class LoadRemoveService { return resultList; } - public Long getViewId(String nature) { + public Long getDataSetId(String nature) { try { String[] split = nature.split(DictWordType.NATURE_SPILT); if (split.length <= 1) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java index dacdfb09e..7c1d2a3e5 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java @@ -7,7 +7,7 @@ public enum TypeEnums { TAG, DOMAIN, ENTITY, - VIEW, + DATASET, MODEL, UNKNOWN; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java similarity index 54% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java index 9e39b73c9..856e5664c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetDetail.java @@ -4,8 +4,8 @@ import lombok.Data; import java.util.List; @Data -public class ViewDetail { +public class DataSetDetail { - private List viewModelConfigs; + private List dataSetModelConfigs; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java similarity index 81% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java index 3e88e0c0e..ceb4e4b06 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java @@ -10,7 +10,7 @@ import java.util.List; @Data @AllArgsConstructor @NoArgsConstructor -public class ViewModelConfig { +public class DataSetModelConfig { private Long id; @@ -20,7 +20,7 @@ public class ViewModelConfig { private List dimensions = Lists.newArrayList(); - public ViewModelConfig(Long id, List dimensions, List metrics) { + public DataSetModelConfig(Long id, List dimensions, List metrics) { this.id = id; this.metrics = metrics; this.dimensions = dimensions; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java index 26223b95d..6b133974d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java @@ -25,8 +25,8 @@ public class QueryParam { private QueryType queryType; private String s2SQL; private String correctS2SQL; - private Long viewId; - private String viewName; + private Long dataSetId; + private String dataSetName; private Set modelIds = new HashSet<>(); private List params = new ArrayList<>(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java index 9329182e8..6123b5ef8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java @@ -16,7 +16,7 @@ public class QueryStat { private String traceId; private Long modelId; - private Long viewId; + private Long dataSetId; private String user; private String createdAt; /** @@ -93,8 +93,8 @@ public class QueryStat { return this; } - public QueryStat setViewId(Long viewId) { - this.viewId = viewId; + public QueryStat setDataSetId(Long dataSetId) { + this.dataSetId = dataSetId; return this; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java index f80f5cda5..aba57cb31 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java @@ -17,7 +17,7 @@ import java.util.List; @NoArgsConstructor public class SchemaElement implements Serializable { - private Long view; + private Long dataSet; private Long model; private Long id; private String name; @@ -41,7 +41,7 @@ public class SchemaElement implements Serializable { return false; } SchemaElement schemaElement = (SchemaElement) o; - return Objects.equal(view, schemaElement.view) && Objects.equal(id, + return Objects.equal(dataSet, schemaElement.dataSet) && Objects.equal(id, schemaElement.id) && Objects.equal(name, schemaElement.name) && Objects.equal(bizName, schemaElement.bizName) && Objects.equal(type, schemaElement.type); @@ -49,7 +49,7 @@ public class SchemaElement implements Serializable { @Override public int hashCode() { - return Objects.hashCode(view, id, name, bizName, type); + return Objects.hashCode(dataSet, id, name, bizName, type); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java index c64f8d4b8..0f410f5cf 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.headless.api.pojo; public enum SchemaElementType { - VIEW, + DATASET, METRIC, DIMENSION, VALUE, diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java index 1cc5b1081..e40c34b8e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.enums; /** * model datasource define type: - * sql_query : view sql begin as select + * sql_query : dataSet sql begin as select * table_query: dbName.tableName */ public enum ModelDefineType { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java new file mode 100644 index 000000000..f0f965670 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetFilterReq.java @@ -0,0 +1,33 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.google.common.collect.Lists; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +@Data +@NoArgsConstructor +public class DataSetFilterReq { + + protected boolean defaultAll = true; + + protected List dataSetIds = Lists.newArrayList(); + + public DataSetFilterReq(Long dataSetId) { + addDataSet(dataSetId); + } + + public List getDataSetIds() { + if (CollectionUtils.isEmpty(dataSetIds) && !defaultAll) { + return Lists.newArrayList(-1L); + } + return dataSetIds; + } + + public void addDataSet(Long dataSetId) { + dataSetIds.add(dataSetId); + } + +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java similarity index 82% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java index c59a15293..e88f2343c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java @@ -2,17 +2,17 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.ViewDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; import lombok.Data; import java.util.List; @Data -public class ViewReq extends SchemaItem { +public class DataSetReq extends SchemaItem { private Long domainId; - private ViewDetail viewDetail; + private DataSetDetail dataSetDetail; private String alias; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryViewReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java similarity index 92% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryViewReq.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java index 2fc840933..f2f0fae56 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryViewReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java @@ -7,17 +7,18 @@ import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.Cache; import com.tencent.supersonic.headless.api.pojo.Param; -import java.util.ArrayList; -import java.util.List; import lombok.Data; import lombok.ToString; +import java.util.ArrayList; +import java.util.List; + @Data @ToString -public class QueryViewReq { +public class QueryDataSetReq { - private Long viewId; - private String viewName; + private Long dataSetId; + private String dataSetName; private String sql; private boolean needAuth = true; private List params = new ArrayList<>(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java index f37caf3dd..5e853c099 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java @@ -27,7 +27,7 @@ public class QueryMultiStructReq extends SemanticQueryReq { if (CollectionUtils.isEmpty(this.getQueryStructReqs())) { return null; } - return this.getQueryStructReqs().get(0).getViewId(); + return this.getQueryStructReqs().get(0).getDataSetId(); } public Cache getCacheInfo() { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java index ddac40577..c2a3c378d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java @@ -12,8 +12,8 @@ public class QuerySqlReq extends SemanticQueryReq { @Override public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"viewId\":") - .append(viewId); + stringBuilder.append("\"dataSetId\":") + .append(dataSetId); stringBuilder.append("\"modelIds\":") .append(modelIds); stringBuilder.append(",\"params\":") diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index a122087a9..4cb35898c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -94,8 +94,8 @@ public class QueryStructReq extends SemanticQueryReq { public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"viewId\":") - .append(viewId); + stringBuilder.append("\"dataSetId\":") + .append(dataSetId); stringBuilder.append("\"modelIds\":") .append(modelIds); stringBuilder.append(",\"groups\":") @@ -127,8 +127,8 @@ public class QueryStructReq extends SemanticQueryReq { @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"viewId\":") - .append(viewId); + sb.append("\"dataSetId\":") + .append(dataSetId); sb.append("\"modelIds\":") .append(modelIds); sb.append(",\"groups\":") @@ -172,7 +172,7 @@ public class QueryStructReq extends SemanticQueryReq { QuerySqlReq result = new QuerySqlReq(); result.setSql(sql); - result.setViewId(this.getViewId()); + result.setDataSetId(this.getDataSetId()); result.setModelIds(this.getModelIdSet()); result.setParams(new ArrayList<>()); return result; @@ -276,11 +276,11 @@ public class QueryStructReq extends SemanticQueryReq { } public String getTableName() { - if (StringUtils.isNotBlank(viewName)) { - return viewName; + if (StringUtils.isNotBlank(dataSetName)) { + return dataSetName; } - if (viewId != null) { - return Constants.TABLE_PREFIX + viewId; + if (dataSetId != null) { + return Constants.TABLE_PREFIX + dataSetId; } return Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_"); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java index be1a6f3af..93721999c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java @@ -5,14 +5,15 @@ import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; import lombok.Data; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + @Data @Slf4j @ToString @@ -32,8 +33,8 @@ public class QueryTagReq extends SemanticQueryReq { @Override public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"viewId\":") - .append(viewId); + stringBuilder.append("\"dataSetId\":") + .append(dataSetId); stringBuilder.append("\"modelIds\":") .append(modelIds); stringBuilder.append(",\"groups\":") diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java index bb08c24c8..b806af12c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java @@ -8,7 +8,7 @@ import java.util.List; @Data public class SchemaFilterReq { - private Long viewId; + private Long dataSetId; private List modelIds = Lists.newArrayList(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java index a9a2dedd5..19976b134 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java @@ -19,9 +19,9 @@ public abstract class SemanticQueryReq { protected boolean needAuth = true; - protected Long viewId; + protected Long dataSetId; - protected String viewName; + protected String dataSetName; protected Set modelIds = new HashSet<>(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java deleted file mode 100644 index af113b435..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.tencent.supersonic.headless.api.pojo.request; - -import com.google.common.collect.Lists; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.springframework.util.CollectionUtils; - -import java.util.List; - -@Data -@NoArgsConstructor -public class ViewFilterReq { - - protected boolean defaultAll = true; - - protected List viewIds = Lists.newArrayList(); - - public ViewFilterReq(Long viewId) { - addView(viewId); - } - - public List getViewIds() { - if (CollectionUtils.isEmpty(viewIds) && !defaultAll) { - return Lists.newArrayList(-1L); - } - return viewIds; - } - - public void addView(Long viewId) { - viewIds.add(viewId); - } - -} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java similarity index 58% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java index 143bae90e..03a5ee005 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.ViewDetail; -import com.tencent.supersonic.headless.api.pojo.ViewModelConfig; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import lombok.Data; import org.springframework.util.CollectionUtils; @@ -14,11 +14,11 @@ import java.util.List; import java.util.stream.Collectors; @Data -public class ViewResp extends SchemaItem { +public class DataSetResp extends SchemaItem { private Long domainId; - private ViewDetail viewDetail; + private DataSetDetail dataSetDetail; private String alias; @@ -31,31 +31,31 @@ public class ViewResp extends SchemaItem { private QueryConfig queryConfig; public List getAllMetrics() { - return getViewModelConfigs().stream().map(ViewModelConfig::getMetrics) + return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics) .flatMap(Collection::stream).collect(Collectors.toList()); } public List getAllDimensions() { - return getViewModelConfigs().stream().map(ViewModelConfig::getDimensions) + return getDataSetModelConfigs().stream().map(DataSetModelConfig::getDimensions) .flatMap(Collection::stream).collect(Collectors.toList()); } public List getAllModels() { - return getViewModelConfigs().stream().map(ViewModelConfig::getId) + return getDataSetModelConfigs().stream().map(DataSetModelConfig::getId) .collect(Collectors.toList()); } public List getAllIncludeAllModels() { - return getViewModelConfigs().stream().filter(ViewModelConfig::isIncludesAll) - .map(ViewModelConfig::getId) + return getDataSetModelConfigs().stream().filter(DataSetModelConfig::isIncludesAll) + .map(DataSetModelConfig::getId) .collect(Collectors.toList()); } - private List getViewModelConfigs() { - if (viewDetail == null || CollectionUtils.isEmpty(viewDetail.getViewModelConfigs())) { + private List getDataSetModelConfigs() { + if (dataSetDetail == null || CollectionUtils.isEmpty(dataSetDetail.getDataSetModelConfigs())) { return Lists.newArrayList(); } - return viewDetail.getViewModelConfigs(); + return dataSetDetail.getDataSetModelConfigs(); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java similarity index 95% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java index 5902454a8..908e0e63d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java @@ -11,7 +11,7 @@ import java.util.List; @Data @AllArgsConstructor @NoArgsConstructor -public class ViewSchemaResp extends ViewResp { +public class DataSetSchemaResp extends DataSetResp { private List metrics = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index d333e51b4..3a470a4ad 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -17,7 +17,7 @@ import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; @NoArgsConstructor public class SemanticSchemaResp { - private Long viewId; + private Long dataSetId; private List modelIds; private SchemaType schemaType; private List metrics = Lists.newArrayList(); @@ -25,14 +25,14 @@ public class SemanticSchemaResp { private List tags = Lists.newArrayList(); private List modelRelas = Lists.newArrayList(); private List modelResps = Lists.newArrayList(); - private ViewResp viewResp; + private DataSetResp dataSetResp; private DatabaseResp databaseResp; public String getSchemaKey() { - if (viewId == null) { + if (dataSetId == null) { return String.format("%s_%s", schemaType, StringUtils.join(modelIds, UNDERLINE)); } - return String.format("%s_%s", schemaType, viewId); + return String.format("%s_%s", schemaType, dataSetId); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DataSetInfoStat.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DataSetInfoStat.java new file mode 100644 index 000000000..153c26cf8 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DataSetInfoStat.java @@ -0,0 +1,22 @@ +package com.tencent.supersonic.headless.core.knowledge; + +import lombok.Builder; +import lombok.Data; +import lombok.ToString; + +import java.io.Serializable; + +@Data +@ToString +@Builder +public class DataSetInfoStat implements Serializable { + + private long dataSetCount; + + private long metricDataSetCount; + + private long dimensionDataSetCount; + + private long dimensionValueDataSetCount; + +} \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java index 7e75f1ffb..eb1adef7c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java @@ -58,7 +58,7 @@ public class SearchService { .collect(Collectors.toList()); for (HanlpMapResult hanlpMapResult : hanlpMapResults) { List natures = hanlpMapResult.getNatures().stream() - .map(nature -> NatureHelper.changeModel2View(nature, modelIdToViewIds)) + .map(nature -> NatureHelper.changeModel2DataSet(nature, modelIdToViewIds)) .flatMap(Collection::stream).collect(Collectors.toList()); hanlpMapResult.setNatures(natures); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java deleted file mode 100644 index 00ba8d2af..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.tencent.supersonic.headless.core.knowledge; - -import lombok.Builder; -import lombok.Data; -import lombok.ToString; - -import java.io.Serializable; - -@Data -@ToString -@Builder -public class ViewInfoStat implements Serializable { - - private long viewCount; - - private long metricViewCount; - - private long dimensionViewCount; - - private long dimensionValueViewCount; - -} \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java index 7f371a139..84658fc3c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java @@ -21,13 +21,13 @@ public class ModelWordBuilder extends BaseWordBuilder { public List doGet(String word, SchemaElement schemaElement) { List result = Lists.newArrayList(); //modelName - DictWord dictWord = buildDictWord(word, schemaElement.getView()); + DictWord dictWord = buildDictWord(word, schemaElement.getDataSet()); result.add(dictWord); //alias List aliasList = schemaElement.getAlias(); if (CollectionUtils.isNotEmpty(aliasList)) { for (String alias : aliasList) { - result.add(buildDictWord(alias, schemaElement.getView())); + result.add(buildDictWord(alias, schemaElement.getDataSet())); } } return result; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java index c4a5b510d..e4c48819d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java @@ -224,7 +224,7 @@ public class HanlpHelper { public static List transform2ApiTerm(Term term, Map> modelIdToViewIds) { List s2Terms = Lists.newArrayList(); - List natures = NatureHelper.changeModel2View(String.valueOf(term.getNature()), modelIdToViewIds); + List natures = NatureHelper.changeModel2DataSet(String.valueOf(term.getNature()), modelIdToViewIds); for (String nature : natures) { S2Term s2Term = new S2Term(); BeanUtils.copyProperties(term, s2Term); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java index 7f88d626b..8daa3f75e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java @@ -5,7 +5,7 @@ import com.hankcs.hanlp.corpus.tag.Nature; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; -import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat; +import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -41,7 +41,7 @@ public class NatureHelper { result = SchemaElementType.ENTITY; break; case VIEW: - result = SchemaElementType.VIEW; + result = SchemaElementType.DATASET; break; case VALUE: result = SchemaElementType.VALUE; @@ -55,12 +55,12 @@ public class NatureHelper { return result; } - private static boolean isViewOrEntity(S2Term term, Integer model) { + private static boolean isDataSetOrEntity(S2Term term, Integer model) { return (DictWordType.NATURE_SPILT + model).equals(term.nature.toString()) || term.nature.toString() .endsWith(DictWordType.ENTITY.getType()); } - public static Integer getViewByNature(Nature nature) { + public static Integer getDataSetByNature(Nature nature) { if (nature.startsWith(DictWordType.NATURE_SPILT)) { String[] dimensionValues = nature.toString().split(DictWordType.NATURE_SPILT); if (StringUtils.isNumeric(dimensionValues[1])) { @@ -70,7 +70,7 @@ public class NatureHelper { return 0; } - public static Long getViewId(String nature) { + public static Long getDataSetId(String nature) { try { String[] split = nature.split(DictWordType.NATURE_SPILT); if (split.length <= 1) { @@ -96,13 +96,13 @@ public class NatureHelper { return null; } - private static Nature changeModel2View(String nature, Long viewId) { + private static Nature changeModel2DataSet(String nature, Long dataSetId) { try { String[] split = nature.split(DictWordType.NATURE_SPILT); if (split.length <= 1) { return null; } - split[1] = String.valueOf(viewId); + split[1] = String.valueOf(dataSetId); return Nature.create(StringUtils.join(split, DictWordType.NATURE_SPILT)); } catch (NumberFormatException e) { log.error("", e); @@ -110,17 +110,17 @@ public class NatureHelper { return null; } - public static List changeModel2View(String nature, Map> modelIdToViewIds) { + public static List changeModel2DataSet(String nature, Map> modelIdToDataSetIds) { Long modelId = getModelId(nature); - List viewIds = modelIdToViewIds.get(modelId); - if (CollectionUtils.isEmpty(viewIds)) { + List dataSetIds = modelIdToDataSetIds.get(modelId); + if (CollectionUtils.isEmpty(dataSetIds)) { return Lists.newArrayList(); } - return viewIds.stream().map(viewId -> String.valueOf(changeModel2View(nature, viewId))) + return dataSetIds.stream().map(dataSetId -> String.valueOf(changeModel2DataSet(nature, dataSetId))) .collect(Collectors.toList()); } - public static boolean isDimensionValueViewId(String nature) { + public static boolean isDimensionValueDataSetId(String nature) { if (StringUtils.isEmpty(nature)) { return false; } @@ -135,21 +135,21 @@ public class NatureHelper { && StringUtils.isNumeric(split[1]); } - public static ViewInfoStat getViewStat(List terms) { - return ViewInfoStat.builder() - .viewCount(getViewCount(terms)) - .dimensionViewCount(getDimensionCount(terms)) - .metricViewCount(getMetricCount(terms)) - .dimensionValueViewCount(getDimensionValueCount(terms)) + public static DataSetInfoStat getDataSetStat(List terms) { + return DataSetInfoStat.builder() + .dataSetCount(getDataSetCount(terms)) + .dimensionDataSetCount(getDimensionCount(terms)) + .metricDataSetCount(getMetricCount(terms)) + .dimensionValueDataSetCount(getDimensionValueCount(terms)) .build(); } - private static long getViewCount(List terms) { - return terms.stream().filter(term -> isViewOrEntity(term, getViewByNature(term.nature))).count(); + private static long getDataSetCount(List terms) { + return terms.stream().filter(term -> isDataSetOrEntity(term, getDataSetByNature(term.nature))).count(); } private static long getDimensionValueCount(List terms) { - return terms.stream().filter(term -> isDimensionValueViewId(term.nature.toString())).count(); + return terms.stream().filter(term -> isDimensionValueDataSetId(term.nature.toString())).count(); } private static long getDimensionCount(List terms) { @@ -169,13 +169,13 @@ public class NatureHelper { * @param terms * @return */ - public static Map> getViewToNatureStat(List terms) { + public static Map> getDataSetToNatureStat(List terms) { Map> modelToNature = new HashMap<>(); terms.stream().filter( term -> term.nature.startsWith(DictWordType.NATURE_SPILT) ).forEach(term -> { DictWordType dictWordType = DictWordType.getNatureType(String.valueOf(term.nature)); - Long model = getViewId(String.valueOf(term.nature)); + Long model = getDataSetId(String.valueOf(term.nature)); Map natureTypeMap = new HashMap<>(); natureTypeMap.put(dictWordType, 1); @@ -196,15 +196,15 @@ public class NatureHelper { return modelToNature; } - public static List selectPossibleViews(List terms) { - Map> modelToNatureStat = getViewToNatureStat(terms); - Integer maxViewTypeSize = modelToNatureStat.entrySet().stream() + public static List selectPossibleDataSets(List terms) { + Map> modelToNatureStat = getDataSetToNatureStat(terms); + Integer maxDataSetTypeSize = modelToNatureStat.entrySet().stream() .max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size()) .orElse(null); - if (Objects.isNull(maxViewTypeSize) || maxViewTypeSize == 0) { + if (Objects.isNull(maxDataSetTypeSize) || maxDataSetTypeSize == 0) { return new ArrayList<>(); } - return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxViewTypeSize) + return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxDataSetTypeSize) .map(entry -> entry.getKey()).collect(Collectors.toList()); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java index 77a4fa7f6..656e8dc48 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java @@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; +import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + @Component @Slf4j @Primary @@ -28,8 +29,8 @@ public class DefaultQueryParser implements QueryParser { public void parse(QueryStatement queryStatement) throws Exception { QueryParam queryParam = queryStatement.getQueryParam(); - if (Objects.isNull(queryStatement.getViewQueryParam())) { - queryStatement.setViewQueryParam(new ViewQueryParam()); + if (Objects.isNull(queryStatement.getDataSetQueryParam())) { + queryStatement.setDataSetQueryParam(new DataSetQueryParam()); } if (Objects.isNull(queryStatement.getMetricQueryParam())) { queryStatement.setMetricQueryParam(new MetricQueryParam()); @@ -41,10 +42,10 @@ public class DefaultQueryParser implements QueryParser { headlessConverter.convert(queryStatement); } } - log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getViewQueryParam(), + log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam()); - if (!queryStatement.getViewQueryParam().getSql().isEmpty()) { - queryStatement = parser(queryStatement.getViewQueryParam(), queryStatement); + if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) { + queryStatement = parser(queryStatement.getDataSetQueryParam(), queryStatement); } else { queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); queryStatement = parser(queryStatement); @@ -61,37 +62,37 @@ public class DefaultQueryParser implements QueryParser { queryStatement.setSql(querySql); } - public QueryStatement parser(ViewQueryParam viewQueryParam, QueryStatement queryStatement) { - log.info("parser MetricReq [{}] ", viewQueryParam); + public QueryStatement parser(DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) { + log.info("parser MetricReq [{}] ", dataSetQueryParam); try { - if (!CollectionUtils.isEmpty(viewQueryParam.getTables())) { + if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) { List tables = new ArrayList<>(); - Boolean isSingleTable = viewQueryParam.getTables().size() == 1; - for (MetricTable metricTable : viewQueryParam.getTables()) { - QueryStatement tableSql = parserSql(metricTable, isSingleTable, viewQueryParam, queryStatement); - if (isSingleTable && Objects.nonNull(tableSql.getViewSimplifySql()) - && !tableSql.getViewSimplifySql().isEmpty()) { - queryStatement.setSql(tableSql.getViewSimplifySql()); - queryStatement.setViewQueryParam(viewQueryParam); + Boolean isSingleTable = dataSetQueryParam.getTables().size() == 1; + for (MetricTable metricTable : dataSetQueryParam.getTables()) { + QueryStatement tableSql = parserSql(metricTable, isSingleTable, dataSetQueryParam, queryStatement); + if (isSingleTable && Objects.nonNull(tableSql.getDataSetQueryParam()) + && !tableSql.getDataSetSimplifySql().isEmpty()) { + queryStatement.setSql(tableSql.getDataSetSimplifySql()); + queryStatement.setDataSetQueryParam(dataSetQueryParam); return queryStatement; } tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()}); } if (!tables.isEmpty()) { String sql = ""; - if (viewQueryParam.isSupportWith()) { + if (dataSetQueryParam.isSupportWith()) { sql = "with " + String.join(",", tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect( - Collectors.toList())) + "\n" + viewQueryParam.getSql(); + Collectors.toList())) + "\n" + dataSetQueryParam.getSql(); } else { - sql = viewQueryParam.getSql(); + sql = dataSetQueryParam.getSql(); for (String[] tb : tables) { sql = StringUtils.replace(sql, tb[0], - "(" + tb[1] + ") " + (viewQueryParam.isWithAlias() ? "" : tb[0]), -1); + "(" + tb[1] + ") " + (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1); } } queryStatement.setSql(sql); - queryStatement.setViewQueryParam(viewQueryParam); + queryStatement.setDataSetQueryParam(dataSetQueryParam); return queryStatement; } } @@ -119,7 +120,7 @@ public class DefaultQueryParser implements QueryParser { } private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, - ViewQueryParam viewQueryParam, + DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) throws Exception { MetricQueryParam metricReq = new MetricQueryParam(); metricReq.setMetrics(metricTable.getMetrics()); @@ -131,11 +132,11 @@ public class DefaultQueryParser implements QueryParser { tableSql.setMetricQueryParam(metricReq); tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); - tableSql.setViewId(queryStatement.getViewId()); + tableSql.setDataSetId(queryStatement.getDataSetId()); tableSql.setSemanticModel(queryStatement.getSemanticModel()); if (isSingleMetricTable) { - tableSql.setViewSql(viewQueryParam.getSql()); - tableSql.setViewAlias(metricTable.getAlias()); + tableSql.setDataSetSql(dataSetQueryParam.getSql()); + tableSql.setDataSetAlias(metricTable.getAlias()); } tableSql = parser(tableSql, metricTable.getAggOption()); if (!tableSql.isOk()) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java index e84ed9e10..e70c9dbac 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java @@ -37,14 +37,14 @@ public class CalciteSqlParser implements SqlParser { queryStatement.setSql(aggBuilder.getSql(engineType)); queryStatement.setSourceId(aggBuilder.getSourceId()); if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize() - && Objects.nonNull(queryStatement.getViewAlias()) && !queryStatement.getViewAlias().isEmpty()) { + && Objects.nonNull(queryStatement.getDataSetAlias()) && !queryStatement.getDataSetAlias().isEmpty()) { // simplify model sql with query sql String simplifySql = aggBuilder.simplify( - getSqlByView(aggBuilder.getSql(engineType), queryStatement.getViewSql(), - queryStatement.getViewAlias()), engineType); + getSqlByDataSet(aggBuilder.getSql(engineType), queryStatement.getDataSetSql(), + queryStatement.getDataSetAlias()), engineType); if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { log.debug("simplifySql [{}]", simplifySql); - queryStatement.setViewSimplifySql(simplifySql); + queryStatement.setDataSetSimplifySql(simplifySql); } } return queryStatement; @@ -62,7 +62,7 @@ public class CalciteSqlParser implements SqlParser { return semanticSchema; } - private String getSqlByView(String sql, String viewSql, String viewAlias) { - return String.format("with %s as (%s) %s", viewAlias, sql, viewSql); + private String getSqlByDataSet(String sql, String dataSetSql, String dataSetAlias) { + return String.format("with %s as (%s) %s", dataSetAlias, sql, dataSetSql); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java index 34380dd40..5ff03cb71 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java @@ -53,16 +53,16 @@ public class SchemaBuilder { .withRowCount(1) .build(); schema.add(MATERIALIZATION_SYS_SOURCE, srcTable); - DataSourceTable viewTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW) + DataSourceTable dataSetTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW) .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT) .withRowCount(1) .build(); - schema.add(MATERIALIZATION_SYS_VIEW, viewTable); + schema.add(MATERIALIZATION_SYS_VIEW, dataSetTable); return rootSchema; } - public static void addSourceView(CalciteSchema viewSchema, String dbSrc, String tbSrc, Set dates, + public static void addSourceView(CalciteSchema dataSetSchema, String dbSrc, String tbSrc, Set dates, Set dimensions, Set metrics) { String tb = tbSrc.toLowerCase(); String db = dbSrc.toLowerCase(); @@ -80,14 +80,14 @@ public class SchemaBuilder { .withRowCount(1) .build(); if (Objects.nonNull(db) && !db.isEmpty()) { - SchemaPlus schemaPlus = viewSchema.plus().getSubSchema(db); + SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db); if (Objects.isNull(schemaPlus)) { - viewSchema.plus().add(db, new AbstractSchema()); - schemaPlus = viewSchema.plus().getSubSchema(db); + dataSetSchema.plus().add(db, new AbstractSchema()); + schemaPlus = dataSetSchema.plus().getSubSchema(db); } schemaPlus.add(tb, srcTable); } else { - viewSchema.add(tb, srcTable); + dataSetSchema.add(tb, srcTable); } } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/ViewExpanderImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/ViewExpanderImpl.java index 6b158e46e..a1f7fe1fb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/ViewExpanderImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/ViewExpanderImpl.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.core.parser.calcite.schema; -import java.util.List; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.type.RelDataType; +import java.util.List; + /** * customize the ViewExpander */ @@ -14,7 +15,7 @@ public class ViewExpanderImpl implements RelOptTable.ViewExpander { @Override public RelRoot expandView(RelDataType rowType, String queryString, List schemaPath, - List viewPath) { + List dataSetPath) { return null; } } \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java index 96e119c94..b3315f3a8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; @@ -13,6 +12,18 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode; +import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -24,16 +35,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.springframework.util.CollectionUtils; @Slf4j public class DataSourceNode extends SemanticNode { @@ -126,9 +127,9 @@ public class DataSourceNode extends SemanticNode { return build(datasource, scope); } EngineType engineType = EngineType.fromString(datasource.getType()); - SqlNode view = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope), + SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope), new SqlNodeList(getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)), SqlParserPos.ZERO); - return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, view); + return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, dataSet); } public static List getExtendField(Set exprList, SqlValidatorScope scope, EngineType engineType) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java index 74526987a..87458bf01 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java @@ -12,11 +12,11 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWriter; /** - * extend node to handle lateral explode view + * extend node to handle lateral explode dataSet */ public class LateralViewExplodeNode extends ExtendNode { - public final String sqlNameView = "view"; + public final String sqlNameView = "dataSet"; public final String sqlNameExplode = "explode"; public LateralViewExplodeNode() { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java index def40fed4..ab70c3cb2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java @@ -34,7 +34,7 @@ import java.util.UUID; import java.util.stream.Collectors; /** - * process the table view from the defined data model schema + * process the table dataSet from the defined data model schema */ @Slf4j public class SourceRender extends Renderer { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index 757b7a82f..49521ceb0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; +import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -42,14 +42,15 @@ public class CalculateAggConverter implements HeadlessConverter { String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql); } - public ViewQueryParam generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version) + public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement, + EngineType engineTypeEnum, String version) throws Exception { QueryParam queryParam = queryStatement.getQueryParam(); // 同环比 if (isRatioAccept(queryParam)) { return generateRatioSqlCommand(queryStatement, engineTypeEnum, version); } - ViewQueryParam sqlCommand = new ViewQueryParam(); + DataSetQueryParam sqlCommand = new DataSetQueryParam(); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); @@ -105,13 +106,13 @@ public class CalculateAggConverter implements HeadlessConverter { @Override public void convert(QueryStatement queryStatement) throws Exception { - ViewQueryParam sqlCommend = queryStatement.getViewQueryParam(); + DataSetQueryParam sqlCommend = queryStatement.getDataSetQueryParam(); Database database = queryStatement.getSemanticModel().getDatabase(); - ViewQueryParam viewQueryParam = generateSqlCommend(queryStatement, + DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement, EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); - sqlCommend.setSql(viewQueryParam.getSql()); - sqlCommend.setTables(viewQueryParam.getTables()); - sqlCommend.setSupportWith(viewQueryParam.isSupportWith()); + sqlCommend.setSql(dataSetQueryParam.getSql()); + sqlCommend.setTables(dataSetQueryParam.getTables()); + sqlCommend.setSupportWith(dataSetQueryParam.isSupportWith()); } /** @@ -128,13 +129,13 @@ public class CalculateAggConverter implements HeadlessConverter { return false; } - public ViewQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum, - String version) + public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum, + String version) throws Exception { QueryParam queryParam = queryStatement.getQueryParam(); check(queryParam); queryStatement.setEnableOptimize(false); - ViewQueryParam sqlCommand = new ViewQueryParam(); + DataSetQueryParam sqlCommand = new DataSetQueryParam(); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java index c13747335..6f1cde8e9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java @@ -27,8 +27,8 @@ public class DefaultDimValueConverter implements HeadlessConverter { @Override public boolean accept(QueryStatement queryStatement) { - return !Objects.isNull(queryStatement.getViewQueryParam()) - && !StringUtils.isBlank(queryStatement.getViewQueryParam().getSql()); + return !Objects.isNull(queryStatement.getDataSetQueryParam()) + && !StringUtils.isBlank(queryStatement.getDataSetQueryParam().getSql()); } @Override @@ -39,14 +39,14 @@ public class DefaultDimValueConverter implements HeadlessConverter { if (CollectionUtils.isEmpty(dimensions)) { return; } - String sql = queryStatement.getViewQueryParam().getSql(); + String sql = queryStatement.getDataSetQueryParam().getSql(); List whereFields = SqlSelectHelper.getWhereFields(sql) .stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(whereFields)) { return; } - MetricTable metricTable = queryStatement.getViewQueryParam() + MetricTable metricTable = queryStatement.getDataSetQueryParam() .getTables().stream().findFirst().orElse(null); List expressions = Lists.newArrayList(); for (Dimension dimension : dimensions) { @@ -63,7 +63,7 @@ public class DefaultDimValueConverter implements HeadlessConverter { } } sql = SqlAddHelper.addWhere(sql, expressions); - queryStatement.getViewQueryParam().setSql(sql); + queryStatement.getDataSetQueryParam().setSql(sql); } } \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java similarity index 90% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java index 97a258664..e6f613651 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java @@ -6,7 +6,7 @@ import lombok.Data; import java.util.List; @Data -public class ViewQueryParam { +public class DataSetQueryParam { private String sql = ""; private List tables; private boolean supportWith = true; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java index 89ada4696..e1c13f3f3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java @@ -6,9 +6,9 @@ import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; -import lombok.NoArgsConstructor; @Data @AllArgsConstructor diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index aada7c820..9c19e0e4f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -11,7 +11,7 @@ import java.util.List; @Data public class QueryStatement { - private Long viewId; + private Long dataSetId; private List modelIds; private String sql = ""; private String sourceId = ""; @@ -19,15 +19,15 @@ public class QueryStatement { private Boolean ok; private QueryParam queryParam; private MetricQueryParam metricQueryParam; - private ViewQueryParam viewQueryParam; + private DataSetQueryParam dataSetQueryParam; private Integer status = 0; private Boolean isS2SQL = false; private List> timeRanges; private Boolean enableOptimize = true; private Triple minMaxTime; - private String viewSql = ""; - private String viewAlias = ""; - private String viewSimplifySql = ""; + private String dataSetSql = ""; + private String dataSetAlias = ""; + private String dataSetSimplifySql = ""; private Boolean enableLimitWrapper = false; private SemanticModel semanticModel; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java index 2f0b4f57b..dbe4c1e81 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java @@ -37,10 +37,10 @@ public class MetricDrillDownChecker { public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { Object[] objects = joinPoint.getArgs(); QueryStatement queryStatement = (QueryStatement) objects[0]; - if (queryStatement.getViewQueryParam() == null) { + if (queryStatement.getDataSetQueryParam() == null) { return joinPoint.proceed(); } - checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getViewQueryParam().getSql()); + checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getDataSetQueryParam().getSql()); return joinPoint.proceed(); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index 5b27827e1..cb92de213 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -30,13 +30,13 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -84,7 +84,7 @@ public class S2DataPermissionAspect { @Autowired private SchemaService schemaService; @Autowired - private ViewService viewService; + private DataSetService dataSetService; @Autowired private AuthService authService; @@ -104,7 +104,7 @@ public class S2DataPermissionAspect { if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) { throw new RuntimeException("please provide user information"); } - List modelIds = getModelsInView(queryReq); + List modelIds = getModelsInDataSet(queryReq); // determine whether admin of the model if (doModelAdmin(user, modelIds)) { @@ -129,21 +129,21 @@ public class S2DataPermissionAspect { // fetch data permission meta information SchemaFilterReq filter = new SchemaFilterReq(); filter.setModelIds(querySqlReq.getModelIds()); - filter.setViewId(querySqlReq.getViewId()); + filter.setDataSetId(querySqlReq.getDataSetId()); SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); - List modelIdInView = semanticSchemaResp.getModelResps().stream() + List modelIdInDataSet = semanticSchemaResp.getModelResps().stream() .map(ModelResp::getId).collect(Collectors.toList()); Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp); - log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege); + log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege); Set sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp); Set sensitiveResReq = res4Privilege.parallelStream() .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); // query user privilege info - AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInView, sensitiveResReq); + AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInDataSet, sensitiveResReq); // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView); + Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet); // if sensitive fields without permission are involved in filter, thrown an exception doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq); @@ -157,7 +157,7 @@ public class S2DataPermissionAspect { if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource); + return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource); } // if the column has no permission, hit * @@ -166,7 +166,7 @@ public class S2DataPermissionAspect { log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet); SemanticQueryResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, need2Apply); - addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply); + addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; } @@ -228,23 +228,23 @@ public class S2DataPermissionAspect { // fetch data permission meta information SchemaFilterReq filter = new SchemaFilterReq(); filter.setModelIds(queryStructReq.getModelIds()); - filter.setViewId(queryStructReq.getViewId()); + filter.setDataSetId(queryStructReq.getDataSetId()); SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); - List modelIdInView = semanticSchemaResp.getModelResps().stream() + List modelIdInDataSet = semanticSchemaResp.getModelResps().stream() .map(ModelResp::getId).collect(Collectors.toList()); Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq); - log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege); + log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege); Set sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp); Set sensitiveResReq = res4Privilege.parallelStream() .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInView, sensitiveResReq); + log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInDataSet, sensitiveResReq); // query user privilege info AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, - modelIdInView, sensitiveResReq); + modelIdInDataSet, sensitiveResReq); // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView); + Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet); // if sensitive fields without permission are involved in filter, thrown an exception doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq); @@ -258,7 +258,7 @@ public class S2DataPermissionAspect { if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource); + return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource); } // if the column has no permission, hit * @@ -266,7 +266,7 @@ public class S2DataPermissionAspect { .collect(Collectors.toSet()); SemanticQueryResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, need2Apply); - addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply); + addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; @@ -557,11 +557,11 @@ public class S2DataPermissionAspect { } } - private List getModelsInView(SemanticQueryReq queryReq) { + private List getModelsInDataSet(SemanticQueryReq queryReq) { List modelIds = queryReq.getModelIds(); - if (queryReq.getViewId() != null) { - ViewResp viewResp = viewService.getView(queryReq.getViewId()); - modelIds = viewResp.getAllModels(); + if (queryReq.getDataSetId() != null) { + DataSetResp dataSetResp = dataSetService.getDataSet(queryReq.getDataSetId()); + modelIds = dataSetResp.getAllModels(); } return modelIds; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java similarity index 89% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java index 37646f512..a6387d2c2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java @@ -7,8 +7,8 @@ import lombok.Data; import java.util.Date; @Data -@TableName("s2_view") -public class ViewDO { +@TableName("s2_data_set") +public class DataSetDO { @TableId(type = IdType.AUTO) private Long id; @@ -25,7 +25,7 @@ public class ViewDO { private String alias; - private String viewDetail; + private String dataSetDetail; private Date createdAt; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java similarity index 75% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java index 79be61213..7259e1c18 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ViewDOMapper extends BaseMapper { +public interface DataSetDOMapper extends BaseMapper { } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java index 95781e556..ffac78b87 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java @@ -48,9 +48,9 @@ public class StatRepositoryImpl implements StatRepository { statInfos.stream().forEach(stat -> { String dimensions = stat.getDimensions(); String metrics = stat.getMetrics(); - if (Objects.nonNull(stat.getViewId())) { - updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getViewId()); - updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getViewId()); + if (Objects.nonNull(stat.getDataSetId())) { + updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getDataSetId()); + updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getDataSetId()); } }); map.forEach((k, v) -> { @@ -69,13 +69,13 @@ public class StatRepositoryImpl implements StatRepository { return statMapper.getStatInfo(itemUseCommend); } - private void updateStatMapInfo(Map map, String dimensions, String type, Long viewId) { + private void updateStatMapInfo(Map map, String dimensions, String type, Long dataSetId) { if (Strings.isNotEmpty(dimensions)) { try { List dimensionList = mapper.readValue(dimensions, new TypeReference>() { }); dimensionList.stream().forEach(dimension -> { - String key = viewId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension; + String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension; if (map.containsKey(key)) { map.put(key, map.get(key) + 1); } else { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java index ca05fc42e..981c3e5af 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java @@ -22,7 +22,7 @@ public class MetaFilter { private Long domainId; - private Long viewId; + private Long dataSetId; private Integer sensitiveLevel; @@ -51,7 +51,7 @@ public class MetaFilter { && Objects.equal(bizName, that.bizName) && Objects.equal( createdBy, that.createdBy) && Objects.equal(modelIds, that.modelIds) && Objects.equal(domainId, that.domainId) && Objects.equal( - viewId, that.viewId) && Objects.equal(sensitiveLevel, that.sensitiveLevel) + dataSetId, that.dataSetId) && Objects.equal(sensitiveLevel, that.sensitiveLevel) && Objects.equal(status, that.status) && Objects.equal(key, that.key) && Objects.equal(ids, that.ids) && Objects.equal( fieldsDepend, that.fieldsDepend); @@ -59,7 +59,7 @@ public class MetaFilter { @Override public int hashCode() { - return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId, viewId, sensitiveLevel, status, key, - ids, fieldsDepend); + return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId, + dataSetId, sensitiveLevel, status, key, ids, fieldsDepend); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java similarity index 59% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java index 2cea09cd6..8c8ce917b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java @@ -2,10 +2,10 @@ package com.tencent.supersonic.headless.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.headless.api.pojo.request.ViewReq; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -21,43 +21,43 @@ import javax.servlet.http.HttpServletResponse; import java.util.List; @RestController -@RequestMapping("/api/semantic/view") -public class ViewController { +@RequestMapping("/api/semantic/dataSet") +public class DataSetController { @Autowired - private ViewService viewService; + private DataSetService dataSetService; @PostMapping - public ViewResp save(@RequestBody ViewReq viewReq, - HttpServletRequest request, HttpServletResponse response) { + public DataSetResp save(@RequestBody DataSetReq dataSetReq, + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return viewService.save(viewReq, user); + return dataSetService.save(dataSetReq, user); } @PutMapping - public ViewResp update(@RequestBody ViewReq viewReq, - HttpServletRequest request, HttpServletResponse response) { + public DataSetResp update(@RequestBody DataSetReq dataSetReq, + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return viewService.update(viewReq, user); + return dataSetService.update(dataSetReq, user); } @GetMapping("/{id}") - public ViewResp getView(@PathVariable("id") Long id) { - return viewService.getView(id); + public DataSetResp getDataSet(@PathVariable("id") Long id) { + return dataSetService.getDataSet(id); } - @GetMapping("/getViewList") - public List getViewList(@RequestParam("domainId") Long domainId) { + @GetMapping("/getDataSetList") + public List getDataSetList(@RequestParam("domainId") Long domainId) { MetaFilter metaFilter = new MetaFilter(); metaFilter.setDomainId(domainId); - return viewService.getViewList(metaFilter); + return dataSetService.getDataSetList(metaFilter); } @DeleteMapping("/{id}") public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - viewService.delete(id, user); + dataSetService.delete(id, user); return true; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java index 349a6b2f3..7d437b260 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java @@ -7,12 +7,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.server.service.SchemaService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; @@ -34,8 +34,8 @@ public class SchemaController { private SchemaService schemaService; @PostMapping - public List fetchViewSchema(@RequestBody ViewFilterReq filter) { - return schemaService.fetchViewSchema(filter); + public List fetchDataSetSchema(@RequestBody DataSetFilterReq filter) { + return schemaService.fetchDataSetSchema(filter); } @GetMapping("/domain/list") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ViewQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/DataSetQueryApiController.java similarity index 73% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ViewQueryApiController.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/DataSetQueryApiController.java index 198493502..4f856c5b4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ViewQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/DataSetQueryApiController.java @@ -2,12 +2,10 @@ package com.tencent.supersonic.headless.server.rest.api; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.QueryService; -import com.tencent.supersonic.headless.server.service.ViewService; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -15,22 +13,25 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + @RestController @RequestMapping("/api/semantic/query") @Slf4j -public class ViewQueryApiController { +public class DataSetQueryApiController { @Autowired - private ViewService viewService; + private DataSetService dataSetService; @Autowired private QueryService queryService; - @PostMapping("/view") - public Object queryByView(@RequestBody QueryViewReq queryViewReq, + @PostMapping("/dataSet") + public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - SemanticQueryReq queryReq = viewService.convert(queryViewReq); + SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq); return queryService.queryByReq(queryReq, user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java new file mode 100644 index 000000000..07ac0043b --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java @@ -0,0 +1,32 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; + +import java.util.List; +import java.util.Map; + +public interface DataSetService { + + DataSetResp save(DataSetReq dataSetReq, User user); + + DataSetResp update(DataSetReq dataSetReq, User user); + + DataSetResp getDataSet(Long id); + + List getDataSetList(MetaFilter metaFilter); + + void delete(Long id, User user); + + Map> getModelIdToDataSetIds(List dataSetIds); + + List getDataSets(User user); + + List getDataSetsInheritAuth(User user, Long domainId); + + SemanticQueryReq convert(QueryDataSetReq queryDataSetReq); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java index 2b3598b33..3e5e92562 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java @@ -12,9 +12,9 @@ public interface KnowledgeService { List getTerms(String text); - List prefixSearch(String key, int limit, Set viewIds); + List prefixSearch(String key, int limit, Set dataSetIds); - List suffixSearch(String key, int limit, Set viewIds); + List suffixSearch(String key, int limit, Set dataSetIds); void updateSemanticKnowledge(List natures); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java index 315f940a2..e24fd8bfb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java @@ -6,6 +6,6 @@ import java.util.List; public interface MetaEmbeddingService { - List retrieveQuery(List viewIds, RetrieveQuery retrieveQuery, int num); + List retrieveQuery(List dataSetIds, RetrieveQuery retrieveQuery, int num); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 2cb90fdac..59b94ca77 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; @@ -17,15 +17,15 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import java.util.List; import java.util.concurrent.ExecutionException; public interface SchemaService { - List fetchViewSchema(ViewFilterReq filter); + List fetchDataSetSchema(DataSetFilterReq filter); List fetchModelSchemaResps(List modelIds); @@ -39,11 +39,11 @@ public interface SchemaService { List getModelList(User user, AuthType authType, Long domainId); - List getViewList(Long domainId); + List getDataSetList(Long domainId); SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq); List getStatInfo(ItemUseReq itemUseReq) throws ExecutionException; - List getDomainViewTree(); + List getDomainDataSetTree(); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java deleted file mode 100644 index 344af1610..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.tencent.supersonic.headless.server.service; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq; -import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewReq; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; - -import java.util.List; -import java.util.Map; - -public interface ViewService { - - ViewResp save(ViewReq viewReq, User user); - - ViewResp update(ViewReq viewReq, User user); - - ViewResp getView(Long id); - - List getViewList(MetaFilter metaFilter); - - void delete(Long id, User user); - - Map> getModelIdToViewIds(List viewIds); - - List getViews(User user); - - List getViewsInheritAuth(User user, Long domainId); - - SemanticQueryReq convert(QueryViewReq queryViewReq); -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java index c017f13c3..ea26c8e13 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java @@ -24,7 +24,7 @@ import java.util.List; @Service public class CanvasServiceImpl { - private CanvasRepository viewInfoRepository; + private CanvasRepository canvasRepository; private ModelService modelService; @@ -32,16 +32,16 @@ public class CanvasServiceImpl { private MetricService metricService; - public CanvasServiceImpl(CanvasRepository viewInfoRepository, ModelService modelService, + public CanvasServiceImpl(CanvasRepository canvasRepository, ModelService modelService, MetricService metricService, DimensionService dimensionService) { - this.viewInfoRepository = viewInfoRepository; + this.canvasRepository = canvasRepository; this.dimensionService = dimensionService; this.metricService = metricService; this.modelService = modelService; } public List getCanvasList(Long domainId) { - return viewInfoRepository.getCanvasList(domainId); + return canvasRepository.getCanvasList(domainId); } public List getCanvasSchema(Long domainId, User user) { @@ -70,20 +70,20 @@ public class CanvasServiceImpl { viewInfoDO.setCreatedBy(user.getName()); viewInfoDO.setUpdatedAt(new Date()); viewInfoDO.setUpdatedBy(user.getName()); - viewInfoRepository.createCanvas(viewInfoDO); + canvasRepository.createCanvas(viewInfoDO); return viewInfoDO; } Long id = canvasReq.getId(); - CanvasDO viewInfoDO = viewInfoRepository.getCanvasById(id); + CanvasDO viewInfoDO = canvasRepository.getCanvasById(id); BeanUtils.copyProperties(canvasReq, viewInfoDO); viewInfoDO.setUpdatedAt(new Date()); viewInfoDO.setUpdatedBy(user.getName()); - viewInfoRepository.updateCanvas(viewInfoDO); + canvasRepository.updateCanvas(viewInfoDO); return viewInfoDO; } public void deleteCanvas(Long id) { - viewInfoRepository.deleteCanvas(id); + canvasRepository.deleteCanvas(id); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java index a41b02a48..56ec65f51 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java @@ -25,7 +25,7 @@ import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Component; @@ -45,16 +45,16 @@ public class CatalogImpl implements Catalog { private final DimensionService dimensionService; private final MetricService metricService; private final ModelRelaService modelRelaService; - private final ViewService viewService; + private final DataSetService dataSetService; private final SchemaService schemaService; public CatalogImpl(DatabaseService databaseService, SchemaService schemaService, - ModelService modelService, DimensionService dimensionService, ViewService viewService, + ModelService modelService, DimensionService dimensionService, DataSetService dataSetService, MetricService metricService, ModelRelaService modelRelaService) { this.databaseService = databaseService; this.modelService = modelService; this.dimensionService = dimensionService; - this.viewService = viewService; + this.dataSetService = dataSetService; this.metricService = metricService; this.modelRelaService = modelRelaService; this.schemaService = schemaService; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetImpl.java similarity index 51% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetImpl.java index 2c4a0d546..31dd472b2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetImpl.java @@ -13,23 +13,23 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.QueryConfig; -import com.tencent.supersonic.headless.api.pojo.ViewDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO; -import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; +import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.BeanUtils; @@ -50,10 +50,10 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -public class ViewServiceImpl - extends ServiceImpl implements ViewService { +public class DataSetImpl + extends ServiceImpl implements DataSetService { - protected final Cache> viewSchemaCache = + protected final Cache> dataSetSchemaCache = CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build(); @Autowired @@ -68,154 +68,156 @@ public class ViewServiceImpl private MetricService metricService; @Override - public ViewResp save(ViewReq viewReq, User user) { - viewReq.createdBy(user.getName()); - ViewDO viewDO = convert(viewReq); - viewDO.setStatus(StatusEnum.ONLINE.getCode()); - ViewResp viewResp = convert(viewDO); - conflictCheck(viewResp); - save(viewDO); - return viewResp; + public DataSetResp save(DataSetReq dataSetReq, User user) { + dataSetReq.createdBy(user.getName()); + DataSetDO dataSetDO = convert(dataSetReq); + dataSetDO.setStatus(StatusEnum.ONLINE.getCode()); + DataSetResp dataSetResp = convert(dataSetDO); + conflictCheck(dataSetResp); + save(dataSetDO); + return dataSetResp; } @Override - public ViewResp update(ViewReq viewReq, User user) { - viewReq.updatedBy(user.getName()); - ViewDO viewDO = convert(viewReq); - ViewResp viewResp = convert(viewDO); - conflictCheck(viewResp); - updateById(viewDO); - return viewResp; + public DataSetResp update(DataSetReq dataSetReq, User user) { + dataSetReq.updatedBy(user.getName()); + DataSetDO dataSetDO = convert(dataSetReq); + DataSetResp dataSetResp = convert(dataSetDO); + conflictCheck(dataSetResp); + updateById(dataSetDO); + return dataSetResp; } @Override - public ViewResp getView(Long id) { - ViewDO viewDO = getById(id); - return convert(viewDO); + public DataSetResp getDataSet(Long id) { + DataSetDO dataSetDO = getById(id); + return convert(dataSetDO); } @Override - public List getViewList(MetaFilter metaFilter) { - QueryWrapper wrapper = new QueryWrapper<>(); + public List getDataSetList(MetaFilter metaFilter) { + QueryWrapper wrapper = new QueryWrapper<>(); if (metaFilter.getDomainId() != null) { - wrapper.lambda().eq(ViewDO::getDomainId, metaFilter.getDomainId()); + wrapper.lambda().eq(DataSetDO::getDomainId, metaFilter.getDomainId()); } if (!CollectionUtils.isEmpty(metaFilter.getIds())) { - wrapper.lambda().in(ViewDO::getId, metaFilter.getIds()); + wrapper.lambda().in(DataSetDO::getId, metaFilter.getIds()); } if (metaFilter.getStatus() != null) { - wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus()); + wrapper.lambda().eq(DataSetDO::getStatus, metaFilter.getStatus()); } - wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode()); + wrapper.lambda().ne(DataSetDO::getStatus, StatusEnum.DELETED.getCode()); return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); } @Override public void delete(Long id, User user) { - ViewDO viewDO = getById(id); - viewDO.setStatus(StatusEnum.DELETED.getCode()); - viewDO.setUpdatedBy(user.getName()); - viewDO.setUpdatedAt(new Date()); - updateById(viewDO); + DataSetDO dataSetDO = getById(id); + dataSetDO.setStatus(StatusEnum.DELETED.getCode()); + dataSetDO.setUpdatedBy(user.getName()); + dataSetDO.setUpdatedAt(new Date()); + updateById(dataSetDO); } @Override - public List getViews(User user) { - List viewResps = getViewList(new MetaFilter()); - return getViewFilterByAuth(viewResps, user); + public List getDataSets(User user) { + List dataSetResps = getDataSetList(new MetaFilter()); + return getDataSetFilterByAuth(dataSetResps, user); } @Override - public List getViewsInheritAuth(User user, Long domainId) { - List viewResps = getViewList(new MetaFilter()); - List inheritAuthFormDomain = getViewFilterByDomainAuth(viewResps, user); - Set viewRespSet = new HashSet<>(inheritAuthFormDomain); - List viewFilterByAuth = getViewFilterByAuth(viewResps, user); - viewRespSet.addAll(viewFilterByAuth); + public List getDataSetsInheritAuth(User user, Long domainId) { + List dataSetResps = getDataSetList(new MetaFilter()); + List inheritAuthFormDomain = getDataSetFilterByDomainAuth(dataSetResps, user); + Set dataSetRespSet = new HashSet<>(inheritAuthFormDomain); + List dataSetFilterByAuth = getDataSetFilterByAuth(dataSetResps, user); + dataSetRespSet.addAll(dataSetFilterByAuth); if (domainId != null && domainId > 0) { - viewRespSet = viewRespSet.stream().filter(modelResp -> + dataSetRespSet = dataSetRespSet.stream().filter(modelResp -> modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet()); } - return viewRespSet.stream().sorted(Comparator.comparingLong(ViewResp::getId)) + return dataSetRespSet.stream().sorted(Comparator.comparingLong(DataSetResp::getId)) .collect(Collectors.toList()); } - private List getViewFilterByAuth(List viewResps, User user) { - return viewResps.stream() - .filter(viewResp -> checkAdminPermission(user, viewResp)) + private List getDataSetFilterByAuth(List dataSetResps, User user) { + return dataSetResps.stream() + .filter(dataSetResp -> checkAdminPermission(user, dataSetResp)) .collect(Collectors.toList()); } - private List getViewFilterByDomainAuth(List viewResps, User user) { + private List getDataSetFilterByDomainAuth(List dataSetResps, User user) { Set domainResps = domainService.getDomainAuthSet(user, AuthType.ADMIN); if (CollectionUtils.isEmpty(domainResps)) { return Lists.newArrayList(); } Set domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet()); - return viewResps.stream().filter(viewResp -> - domainIds.contains(viewResp.getDomainId())).collect(Collectors.toList()); + return dataSetResps.stream().filter(dataSetResp -> + domainIds.contains(dataSetResp.getDomainId())).collect(Collectors.toList()); } - private ViewResp convert(ViewDO viewDO) { - ViewResp viewResp = new ViewResp(); - BeanMapper.mapper(viewDO, viewResp); - viewResp.setViewDetail(JSONObject.parseObject(viewDO.getViewDetail(), ViewDetail.class)); - if (viewDO.getQueryConfig() != null) { - viewResp.setQueryConfig(JSONObject.parseObject(viewDO.getQueryConfig(), QueryConfig.class)); + private DataSetResp convert(DataSetDO dataSetDO) { + DataSetResp dataSetResp = new DataSetResp(); + BeanMapper.mapper(dataSetDO, dataSetResp); + dataSetResp.setDataSetDetail(JSONObject.parseObject(dataSetDO.getDataSetDetail(), DataSetDetail.class)); + if (dataSetDO.getQueryConfig() != null) { + dataSetResp.setQueryConfig(JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class)); } - viewResp.setAdmins(StringUtils.isBlank(viewDO.getAdmin()) - ? Lists.newArrayList() : Arrays.asList(viewDO.getAdmin().split(","))); - viewResp.setAdminOrgs(StringUtils.isBlank(viewDO.getAdminOrg()) - ? Lists.newArrayList() : Arrays.asList(viewDO.getAdminOrg().split(","))); - viewResp.setTypeEnum(TypeEnums.VIEW); - return viewResp; + dataSetResp.setAdmins(StringUtils.isBlank(dataSetDO.getAdmin()) + ? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdmin().split(","))); + dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) + ? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(","))); + dataSetResp.setTypeEnum(TypeEnums.DATASET); + return dataSetResp; } - private ViewDO convert(ViewReq viewReq) { - ViewDO viewDO = new ViewDO(); - BeanMapper.mapper(viewReq, viewDO); - viewDO.setViewDetail(JSONObject.toJSONString(viewReq.getViewDetail())); - viewDO.setQueryConfig(JSONObject.toJSONString(viewReq.getQueryConfig())); - return viewDO; + private DataSetDO convert(DataSetReq dataSetReq) { + DataSetDO dataSetDO = new DataSetDO(); + BeanMapper.mapper(dataSetReq, dataSetDO); + dataSetDO.setDataSetDetail(JSONObject.toJSONString(dataSetReq.getDataSetDetail())); + dataSetDO.setQueryConfig(JSONObject.toJSONString(dataSetReq.getQueryConfig())); + return dataSetDO; } - public SemanticQueryReq convert(QueryViewReq queryViewReq) { + public SemanticQueryReq convert(QueryDataSetReq queryDataSetReq) { SemanticQueryReq queryReq = new QueryStructReq(); - if (StringUtils.isNotBlank(queryViewReq.getSql())) { + if (StringUtils.isNotBlank(queryDataSetReq.getSql())) { queryReq = new QuerySqlReq(); } - BeanUtils.copyProperties(queryViewReq, queryReq); + BeanUtils.copyProperties(queryDataSetReq, queryReq); return queryReq; } - public static boolean checkAdminPermission(User user, ViewResp viewResp) { - List admins = viewResp.getAdmins(); + public static boolean checkAdminPermission(User user, DataSetResp dataSetResp) { + List admins = dataSetResp.getAdmins(); if (user.isSuperAdmin()) { return true; } String userName = user.getName(); - return admins.contains(userName) || viewResp.getCreatedBy().equals(userName); + return admins.contains(userName) || dataSetResp.getCreatedBy().equals(userName); } @Override - public Map> getModelIdToViewIds(List viewIds) { + public Map> getModelIdToDataSetIds(List dataSetIds) { MetaFilter metaFilter = new MetaFilter(); metaFilter.setStatus(StatusEnum.ONLINE.getCode()); - metaFilter.setIds(viewIds); - List viewList = viewSchemaCache.getIfPresent(metaFilter); - if (CollectionUtils.isEmpty(viewList)) { - viewList = getViewList(metaFilter); - viewSchemaCache.put(metaFilter, viewList); + metaFilter.setIds(dataSetIds); + List dataSetList = dataSetSchemaCache.getIfPresent(metaFilter); + if (CollectionUtils.isEmpty(dataSetList)) { + dataSetList = getDataSetList(metaFilter); + dataSetSchemaCache.put(metaFilter, dataSetList); } - return viewList.stream() + return dataSetList.stream() .flatMap( - viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId()))) - .collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList()))); + dataSetResp -> dataSetResp.getAllModels().stream().map(modelId -> + Pair.of(modelId, dataSetResp.getId()))) + .collect(Collectors.groupingBy(Pair::getLeft, + Collectors.mapping(Pair::getRight, Collectors.toList()))); } - private void conflictCheck(ViewResp viewResp) { - List allDimensionIds = viewResp.getAllDimensions(); - List allMetricIds = viewResp.getAllMetrics(); + private void conflictCheck(DataSetResp dataSetResp) { + List allDimensionIds = dataSetResp.getAllDimensions(); + List allMetricIds = dataSetResp.getAllMetrics(); MetaFilter metaFilter = new MetaFilter(); if (!CollectionUtils.isEmpty(allDimensionIds)) { metaFilter.setIds(allDimensionIds); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index e3182e93f..e851f1b9c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -21,29 +21,23 @@ import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -51,6 +45,13 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + @Service @Slf4j public class DimensionServiceImpl implements DimensionService { @@ -66,7 +67,7 @@ public class DimensionServiceImpl implements DimensionService { private ModelRelaService modelRelaService; - private ViewService viewService; + private DataSetService dataSetService; @Autowired private ApplicationEventPublisher eventPublisher; @@ -77,13 +78,13 @@ public class DimensionServiceImpl implements DimensionService { ChatGptHelper chatGptHelper, DatabaseService databaseService, ModelRelaService modelRelaService, - ViewService viewService) { + DataSetService dataSetService) { this.modelService = modelService; this.dimensionRepository = dimensionRepository; this.chatGptHelper = chatGptHelper; this.databaseService = databaseService; this.modelRelaService = modelRelaService; - this.viewService = viewService; + this.dataSetService = dataSetService; } @Override @@ -232,9 +233,9 @@ public class DimensionServiceImpl implements DimensionService { if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) { return filterByField(dimensionResps, metaFilter.getFieldsDepend()); } - if (metaFilter.getViewId() != null) { - ViewResp viewResp = viewService.getView(metaFilter.getViewId()); - return DimensionConverter.filterByView(dimensionResps, viewResp); + if (metaFilter.getDataSetId() != null) { + DataSetResp dataSetResp = dataSetService.getDataSet(metaFilter.getDataSetId()); + return DimensionConverter.filterByDataSet(dimensionResps, dataSetResp); } return dimensionResps; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java index fb4df6514..6ed24faea 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java @@ -10,12 +10,12 @@ import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.DomainUpdateReq; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO; import com.tencent.supersonic.headless.server.persistence.repository.DomainRepository; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.utils.DomainConvert; import lombok.extern.slf4j.Slf4j; import org.assertj.core.util.Sets; @@ -43,17 +43,17 @@ public class DomainServiceImpl implements DomainService { private final DomainRepository domainRepository; private final ModelService modelService; private final UserService userService; - private final ViewService viewService; + private final DataSetService dataSetService; public DomainServiceImpl(DomainRepository domainRepository, @Lazy ModelService modelService, UserService userService, - @Lazy ViewService viewService) { + @Lazy DataSetService dataSetService) { this.domainRepository = domainRepository; this.modelService = modelService; this.userService = userService; - this.viewService = viewService; + this.dataSetService = dataSetService; } @Override @@ -104,9 +104,9 @@ public class DomainServiceImpl implements DomainService { List domainIds = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList()); domainWithAuthAll.addAll(getParentDomain(domainIds)); } - List viewResps = viewService.getViews(user); - if (!CollectionUtils.isEmpty(viewResps)) { - List domainIds = viewResps.stream().map(ViewResp::getDomainId).collect(Collectors.toList()); + List dataSetResps = dataSetService.getDataSets(user); + if (!CollectionUtils.isEmpty(dataSetResps)) { + List domainIds = dataSetResps.stream().map(DataSetResp::getDomainId).collect(Collectors.toList()); domainWithAuthAll.addAll(getParentDomain(domainIds)); } return new ArrayList<>(domainWithAuthAll).stream() @@ -125,7 +125,7 @@ public class DomainServiceImpl implements DomainService { } if (authTypeEnum.equals(AuthType.VISIBLE)) { domainWithAuth = domainResps.stream() - .filter(domainResp -> checkViewerPermission(orgIds, user, domainResp)) + .filter(domainResp -> checkDataSeterPermission(orgIds, user, domainResp)) .collect(Collectors.toList()); } List domainIds = domainWithAuth.stream().map(DomainResp::getId) @@ -253,7 +253,7 @@ public class DomainServiceImpl implements DomainService { return false; } - private boolean checkViewerPermission(Set orgIds, User user, DomainResp domainDesc) { + private boolean checkDataSeterPermission(Set orgIds, User user, DomainResp domainDesc) { List admins = domainDesc.getAdmins(); List viewers = domainDesc.getViewers(); List adminOrgs = domainDesc.getAdminOrgs(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java index 67b85cca6..9ef611caa 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.server.service.KnowledgeService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -21,10 +21,10 @@ import java.util.stream.Collectors; @Slf4j public class KnowledgeServiceImpl implements KnowledgeService { - private final ViewService viewService; + private final DataSetService dataSetService; - public KnowledgeServiceImpl(ViewService viewService) { - this.viewService = viewService; + public KnowledgeServiceImpl(DataSetService dataSetService) { + this.dataSetService = dataSetService; } @Override @@ -69,25 +69,25 @@ public class KnowledgeServiceImpl implements KnowledgeService { @Override public List getTerms(String text) { - Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>()); - return HanlpHelper.getTerms(text, modelIdToViewIds); + Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>()); + return HanlpHelper.getTerms(text, modelIdToDataSetIds); } @Override - public List prefixSearch(String key, int limit, Set viewIds) { - Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds)); - return prefixSearchByModel(key, limit, modelIdToViewIds); + public List prefixSearch(String key, int limit, Set dataSetIds) { + Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds)); + return prefixSearchByModel(key, limit, modelIdToDataSetIds); } public List prefixSearchByModel(String key, int limit, - Map> modelIdToViewIds) { - return SearchService.prefixSearch(key, limit, modelIdToViewIds); + Map> modelIdToDataSetIds) { + return SearchService.prefixSearch(key, limit, modelIdToDataSetIds); } @Override - public List suffixSearch(String key, int limit, Set viewIds) { - Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds)); - return suffixSearchByModel(key, limit, modelIdToViewIds.keySet()); + public List suffixSearch(String key, int limit, Set dataSetIds) { + Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds)); + return suffixSearchByModel(key, limit, modelIdToDataSetIds.keySet()); } public List suffixSearchByModel(String key, int limit, Set models) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java index b3db16bad..24f0e7b32 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java @@ -7,8 +7,14 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.MetaEmbeddingService; -import com.tencent.supersonic.headless.server.service.ViewService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -17,11 +23,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -32,13 +33,13 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService { private EmbeddingConfig embeddingConfig; @Autowired - private ViewService viewService; + private DataSetService dataSetService; @Override - public List retrieveQuery(List viewIds, RetrieveQuery retrieveQuery, int num) { - // viewIds->modelIds - Map> modelIdToViewIds = viewService.getModelIdToViewIds(viewIds); - Set allModels = modelIdToViewIds.keySet(); + public List retrieveQuery(List dataSetIds, RetrieveQuery retrieveQuery, int num) { + // dataSetIds->modelIds + Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(dataSetIds); + Set allModels = modelIdToDataSetIds.keySet(); if (CollectionUtils.isNotEmpty(allModels) && allModels.size() == 1) { Map filterCondition = new HashMap<>(); @@ -69,16 +70,16 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService { } return !allModels.contains(modelId); }); - //add viewId + //add dataSetId retrievals = retrievals.stream().flatMap(retrieval -> { Long modelId = Retrieval.getLongId(retrieval.getMetadata().get("modelId")); - List viewIdsByModelId = modelIdToViewIds.get(modelId); - if (!CollectionUtils.isEmpty(viewIdsByModelId)) { + List dataSetIdsByModelId = modelIdToDataSetIds.get(modelId); + if (!CollectionUtils.isEmpty(dataSetIdsByModelId)) { Set result = new HashSet<>(); - for (Long viewId : viewIdsByModelId) { + for (Long dataSetId : dataSetIdsByModelId) { Retrieval retrievalNew = new Retrieval(); BeanUtils.copyProperties(retrieval, retrievalNew); - retrievalNew.getMetadata().putIfAbsent("viewId", viewId + Constants.UNDERLINE); + retrievalNew.getMetadata().putIfAbsent("dataSetId", dataSetId + Constants.UNDERLINE); result.add(retrievalNew); } return result.stream(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 2de884972..b2a612954 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -29,10 +29,10 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; @@ -43,13 +43,19 @@ import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import com.tencent.supersonic.headless.server.pojo.ModelCluster; import com.tencent.supersonic.headless.server.service.CollectService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -60,11 +66,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service @Slf4j @@ -80,7 +81,7 @@ public class MetricServiceImpl implements MetricService { private CollectService collectService; - private ViewService viewService; + private DataSetService dataSetService; private ApplicationEventPublisher eventPublisher; @@ -88,7 +89,7 @@ public class MetricServiceImpl implements MetricService { ModelService modelService, ChatGptHelper chatGptHelper, CollectService collectService, - ViewService viewService, + DataSetService dataSetService, ApplicationEventPublisher eventPublisher, DimensionService dimensionService) { this.metricRepository = metricRepository; @@ -96,7 +97,7 @@ public class MetricServiceImpl implements MetricService { this.chatGptHelper = chatGptHelper; this.eventPublisher = eventPublisher; this.collectService = collectService; - this.viewService = viewService; + this.dataSetService = dataSetService; this.dimensionService = dimensionService; } @@ -232,9 +233,9 @@ public class MetricServiceImpl implements MetricService { if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) { return filterByField(metricResps, metaFilter.getFieldsDepend()); } - if (metaFilter.getViewId() != null) { - ViewResp viewResp = viewService.getView(metaFilter.getViewId()); - return MetricConverter.filterByView(metricResps, viewResp); + if (metaFilter.getDataSetId() != null) { + DataSetResp dataSetResp = dataSetService.getDataSet(metaFilter.getDataSetId()); + return MetricConverter.filterByDataSet(metricResps, dataSetResp); } return metricResps; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index f674656d7..0469fa104 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -25,7 +25,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; @@ -37,7 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.utils.ModelConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; @@ -76,7 +76,7 @@ public class ModelServiceImpl implements ModelService { private UserService userService; - private ViewService viewService; + private DataSetService dataSetService; private DateInfoRepository dateInfoRepository; @@ -86,7 +86,7 @@ public class ModelServiceImpl implements ModelService { @Lazy MetricService metricService, DomainService domainService, UserService userService, - ViewService viewService, + DataSetService dataSetService, DateInfoRepository dateInfoRepository) { this.modelRepository = modelRepository; this.databaseService = databaseService; @@ -94,7 +94,7 @@ public class ModelServiceImpl implements ModelService { this.metricService = metricService; this.domainService = domainService; this.userService = userService; - this.viewService = viewService; + this.dataSetService = dataSetService; this.dateInfoRepository = dateInfoRepository; } @@ -145,9 +145,9 @@ public class ModelServiceImpl implements ModelService { ModelFilter modelFilter = new ModelFilter(); BeanUtils.copyProperties(metaFilter, modelFilter); List modelResps = ModelConverter.convertList(modelRepository.getModelList(modelFilter)); - if (modelFilter.getViewId() != null) { - ViewResp viewResp = viewService.getView(modelFilter.getViewId()); - return modelResps.stream().filter(modelResp -> viewResp.getAllModels().contains(modelResp.getId())) + if (modelFilter.getDataSetId() != null) { + DataSetResp dataSetResp = dataSetService.getDataSet(modelFilter.getDataSetId()); + return modelResps.stream().filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId())) .collect(Collectors.toList()); } return modelResps; @@ -329,7 +329,7 @@ public class ModelServiceImpl implements ModelService { } if (authTypeEnum.equals(AuthType.VISIBLE)) { modelWithAuth = modelResps.stream() - .filter(domainResp -> checkViewerPermission(orgIds, user, domainResp)) + .filter(domainResp -> checkDataSeterPermission(orgIds, user, domainResp)) .collect(Collectors.toList()); } return modelWithAuth; @@ -456,7 +456,7 @@ public class ModelServiceImpl implements ModelService { return false; } - public static boolean checkViewerPermission(Set orgIds, User user, ModelResp modelResp) { + public static boolean checkDataSeterPermission(Set orgIds, User user, ModelResp modelResp) { if (checkAdminPermission(orgIds, user, modelResp)) { return true; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index 2971915f9..469d02d66 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -51,17 +51,18 @@ import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.utils.TagReqConverter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import javax.servlet.http.HttpServletRequest; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.stereotype.Service; @Service @@ -176,7 +177,7 @@ public class QueryServiceImpl implements QueryService { queryStatement.setQueryParam(queryParam); queryStatement.setIsS2SQL(false); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - queryStatement.setViewId(queryStructReq.getViewId()); + queryStatement.setDataSetId(queryStructReq.getDataSetId()); queryStatement.setSemanticSchemaResp(semanticSchemaResp); SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp); queryStatement.setSemanticModel(semanticModel); @@ -216,7 +217,7 @@ public class QueryServiceImpl implements QueryService { private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); - schemaFilterReq.setViewId(semanticQueryReq.getViewId()); + schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId()); schemaFilterReq.setModelIds(semanticQueryReq.getModelIds()); return schemaFilterReq; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index d900d96c6..fcfa6fd9c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; @@ -30,8 +30,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.service.DimensionService; @@ -41,7 +41,7 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.service.TagService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.StatUtils; @@ -68,7 +68,7 @@ public class SchemaServiceImpl implements SchemaService { protected final Cache> itemUseCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); - protected final Cache> viewSchemaCache = + protected final Cache> dataSetSchemaCache = CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build(); protected final Cache semanticSchemaCache = @@ -79,7 +79,7 @@ public class SchemaServiceImpl implements SchemaService { private final DimensionService dimensionService; private final MetricService metricService; private final DomainService domainService; - private final ViewService viewService; + private final DataSetService dataSetService; private final ModelRelaService modelRelaService; private final TagService tagService; @@ -87,14 +87,14 @@ public class SchemaServiceImpl implements SchemaService { DimensionService dimensionService, MetricService metricService, DomainService domainService, - ViewService viewService, + DataSetService dataSetService, ModelRelaService modelRelaService, StatUtils statUtils, TagService tagService) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; this.domainService = domainService; - this.viewService = viewService; + this.dataSetService = dataSetService; this.modelRelaService = modelRelaService; this.statUtils = statUtils; this.tagService = tagService; @@ -102,30 +102,30 @@ public class SchemaServiceImpl implements SchemaService { @SneakyThrows @Override - public List fetchViewSchema(ViewFilterReq filter) { - List viewList = viewSchemaCache.getIfPresent(filter); - if (CollectionUtils.isEmpty(viewList)) { - viewList = buildViewSchema(filter); - viewSchemaCache.put(filter, viewList); + public List fetchDataSetSchema(DataSetFilterReq filter) { + List dataSetList = dataSetSchemaCache.getIfPresent(filter); + if (CollectionUtils.isEmpty(dataSetList)) { + dataSetList = buildDataSetSchema(filter); + dataSetSchemaCache.put(filter, dataSetList); } - return viewList; + return dataSetList; } - public ViewSchemaResp fetchViewSchema(Long viewId) { - if (viewId == null) { + public DataSetSchemaResp fetchDataSetSchema(Long dataSetId) { + if (dataSetId == null) { return null; } - return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null); + return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst().orElse(null); } - public List buildViewSchema(ViewFilterReq filter) { - List viewSchemaResps = new ArrayList<>(); - List viewIds = filter.getViewIds(); + public List buildDataSetSchema(DataSetFilterReq filter) { + List dataSetSchemaResps = new ArrayList<>(); + List dataSetIds = filter.getDataSetIds(); MetaFilter metaFilter = new MetaFilter(); metaFilter.setStatus(StatusEnum.ONLINE.getCode()); - metaFilter.setIds(viewIds); - List viewResps = viewService.getViewList(metaFilter); - List modelIds = viewResps.stream().map(ViewResp::getAllModels) + metaFilter.setIds(dataSetIds); + List dataSetResps = dataSetService.getDataSetList(metaFilter); + List modelIds = dataSetResps.stream().map(DataSetResp::getAllModels) .flatMap(Collection::stream).collect(Collectors.toList()); metaFilter.setModelIds(modelIds); metaFilter.setIds(Lists.newArrayList()); @@ -133,26 +133,26 @@ public class SchemaServiceImpl implements SchemaService { List dimensionResps = dimensionService.getDimensions(metaFilter); metaFilter.setIds(modelIds); List modelResps = modelService.getModelList(metaFilter); - Map viewRespMap = getViewMap(viewResps); - for (Long viewId : viewRespMap.keySet()) { - ViewResp viewResp = viewRespMap.get(viewId); - if (viewResp == null || !StatusEnum.ONLINE.getCode().equals(viewResp.getStatus())) { + Map dataSetRespMap = getDataSetMap(dataSetResps); + for (Long dataSetId : dataSetRespMap.keySet()) { + DataSetResp dataSetResp = dataSetRespMap.get(dataSetId); + if (dataSetResp == null || !StatusEnum.ONLINE.getCode().equals(dataSetResp.getStatus())) { continue; } - List metricSchemaResps = MetricConverter.filterByView(metricResps, viewResp) + List metricSchemaResps = MetricConverter.filterByDataSet(metricResps, dataSetResp) .stream().map(this::convert).collect(Collectors.toList()); - List dimSchemaResps = DimensionConverter.filterByView(dimensionResps, viewResp) + List dimSchemaResps = DimensionConverter.filterByDataSet(dimensionResps, dataSetResp) .stream().map(this::convert).collect(Collectors.toList()); - ViewSchemaResp viewSchemaResp = new ViewSchemaResp(); - BeanUtils.copyProperties(viewResp, viewSchemaResp); - viewSchemaResp.setDimensions(dimSchemaResps); - viewSchemaResp.setMetrics(metricSchemaResps); - viewSchemaResp.setModelResps(modelResps.stream().filter(modelResp -> - viewResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList())); - viewSchemaResps.add(viewSchemaResp); + DataSetSchemaResp dataSetSchemaResp = new DataSetSchemaResp(); + BeanUtils.copyProperties(dataSetResp, dataSetSchemaResp); + dataSetSchemaResp.setDimensions(dimSchemaResps); + dataSetSchemaResp.setMetrics(metricSchemaResps); + dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp -> + dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList())); + dataSetSchemaResps.add(dataSetSchemaResp); } - fillStaticInfo(viewSchemaResps); - return viewSchemaResps; + fillStaticInfo(dataSetSchemaResps); + return dataSetSchemaResps; } public List fetchModelSchemaResps(List modelIds) { @@ -191,7 +191,7 @@ public class SchemaServiceImpl implements SchemaService { } - private void fillCnt(List viewSchemaResps, List statInfos) { + private void fillCnt(List dataSetSchemaResps, List statInfos) { Map typeIdAndStatPair = statInfos.stream() .collect(Collectors.toMap( @@ -199,15 +199,15 @@ public class SchemaServiceImpl implements SchemaService { itemUseInfo -> itemUseInfo, (item1, item2) -> item1)); log.debug("typeIdAndStatPair:{}", typeIdAndStatPair); - for (ViewSchemaResp viewSchemaResp : viewSchemaResps) { - fillDimCnt(viewSchemaResp, typeIdAndStatPair); - fillMetricCnt(viewSchemaResp, typeIdAndStatPair); + for (DataSetSchemaResp dataSetSchemaResp : dataSetSchemaResps) { + fillDimCnt(dataSetSchemaResp, typeIdAndStatPair); + fillMetricCnt(dataSetSchemaResp, typeIdAndStatPair); } } - private void fillMetricCnt(ViewSchemaResp viewSchemaResp, Map typeIdAndStatPair) { - List metrics = viewSchemaResp.getMetrics(); - if (CollectionUtils.isEmpty(viewSchemaResp.getMetrics())) { + private void fillMetricCnt(DataSetSchemaResp dataSetSchemaResp, Map typeIdAndStatPair) { + List metrics = dataSetSchemaResp.getMetrics(); + if (CollectionUtils.isEmpty(dataSetSchemaResp.getMetrics())) { return; } @@ -220,12 +220,12 @@ public class SchemaServiceImpl implements SchemaService { } }); } - viewSchemaResp.setMetrics(metrics); + dataSetSchemaResp.setMetrics(metrics); } - private void fillDimCnt(ViewSchemaResp viewSchemaResp, Map typeIdAndStatPair) { - List dimensions = viewSchemaResp.getDimensions(); - if (CollectionUtils.isEmpty(viewSchemaResp.getDimensions())) { + private void fillDimCnt(DataSetSchemaResp dataSetSchemaResp, Map typeIdAndStatPair) { + List dimensions = dataSetSchemaResp.getDimensions(); + if (CollectionUtils.isEmpty(dataSetSchemaResp.getDimensions())) { return; } if (!CollectionUtils.isEmpty(dimensions)) { @@ -237,7 +237,7 @@ public class SchemaServiceImpl implements SchemaService { } }); } - viewSchemaResp.setDimensions(dimensions); + dataSetSchemaResp.setDimensions(dimensions); } @Override @@ -273,20 +273,20 @@ public class SchemaServiceImpl implements SchemaService { } @Override - public List getViewList(Long domainId) { + public List getDataSetList(Long domainId) { MetaFilter metaFilter = new MetaFilter(); metaFilter.setDomainId(domainId); - return viewService.getViewList(metaFilter); + return dataSetService.getDataSetList(metaFilter); } public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) { SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp(); - semanticSchemaResp.setViewId(schemaFilterReq.getViewId()); + semanticSchemaResp.setDataSetId(schemaFilterReq.getDataSetId()); semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds()); - if (schemaFilterReq.getViewId() != null) { - ViewSchemaResp viewSchemaResp = fetchViewSchema(schemaFilterReq.getViewId()); - BeanUtils.copyProperties(viewSchemaResp, semanticSchemaResp); - List modelIds = viewSchemaResp.getAllModels(); + if (schemaFilterReq.getDataSetId() != null) { + DataSetSchemaResp dataSetSchemaResp = fetchDataSetSchema(schemaFilterReq.getDataSetId()); + BeanUtils.copyProperties(dataSetSchemaResp, semanticSchemaResp); + List modelIds = dataSetSchemaResp.getAllModels(); MetaFilter metaFilter = new MetaFilter(); metaFilter.setIds(modelIds); List modelList = modelService.getModelList(metaFilter); @@ -343,7 +343,7 @@ public class SchemaServiceImpl implements SchemaService { } @Override - public List getDomainViewTree() { + public List getDomainDataSetTree() { List domainResps = domainService.getDomainList(); List itemResps = domainResps.stream().map(domain -> new ItemResp(domain.getId(), domain.getParentId(), domain.getName(), TypeEnums.DOMAIN)) @@ -357,36 +357,36 @@ public class SchemaServiceImpl implements SchemaService { } parentItem.getChildren().add(itemResp); } - List viewResps = viewService.getViewList(new MetaFilter()); - for (ViewResp viewResp : viewResps) { - ItemResp itemResp = itemRespMap.get(viewResp.getDomainId()); + List dataSetResps = dataSetService.getDataSetList(new MetaFilter()); + for (DataSetResp dataSetResp : dataSetResps) { + ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId()); if (itemResp != null) { - ItemResp view = new ItemResp(viewResp.getId(), viewResp.getDomainId(), - viewResp.getName(), TypeEnums.VIEW); - itemResp.getChildren().add(view); + ItemResp dataSet = new ItemResp(dataSetResp.getId(), dataSetResp.getDomainId(), + dataSetResp.getName(), TypeEnums.DATASET); + itemResp.getChildren().add(dataSet); } } return itemResps.stream().filter(itemResp -> itemResp.getParentId() == 0) .collect(Collectors.toList()); } - private void fillStaticInfo(List viewSchemaResps) { - List viewIds = viewSchemaResps.stream() - .map(ViewSchemaResp::getId).collect(Collectors.toList()); + private void fillStaticInfo(List dataSetSchemaResps) { + List dataSetIds = dataSetSchemaResps.stream() + .map(DataSetSchemaResp::getId).collect(Collectors.toList()); ItemUseReq itemUseReq = new ItemUseReq(); - itemUseReq.setModelIds(viewIds); + itemUseReq.setModelIds(dataSetIds); List statInfos = getStatInfo(itemUseReq); log.debug("statInfos:{}", statInfos); - fillCnt(viewSchemaResps, statInfos); + fillCnt(dataSetSchemaResps, statInfos); } - private Map getViewMap(List viewResps) { - if (CollectionUtils.isEmpty(viewResps)) { + private Map getDataSetMap(List dataSetResps) { + if (CollectionUtils.isEmpty(dataSetResps)) { return new HashMap<>(); } - return viewResps.stream().collect( - Collectors.toMap(ViewResp::getId, viewResp -> viewResp)); + return dataSetResps.stream().collect( + Collectors.toMap(DataSetResp::getId, dataSetResp -> dataSetResp)); } private DimSchemaResp convert(DimensionResp dimensionResp) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index 72aa612a0..aae9b3462 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; @@ -79,10 +79,10 @@ public class DimensionConverter { return dimensionResp; } - public static List filterByView(List dimensionResps, ViewResp viewResp) { + public static List filterByDataSet(List dimensionResps, DataSetResp dataSetResp) { return dimensionResps.stream().filter(dimensionResp -> - viewResp.getAllDimensions().contains(dimensionResp.getId()) - || viewResp.getAllIncludeAllModels().contains(dimensionResp.getModelId())) + dataSetResp.getAllDimensions().contains(dimensionResp.getId()) + || dataSetResp.getAllIncludeAllModels().contains(dimensionResp.getModelId())) .collect(Collectors.toList()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index 98cbbea9e..56e491c7c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import org.springframework.beans.BeanUtils; @@ -99,10 +99,10 @@ public class MetricConverter { return metricResp; } - public static List filterByView(List metricResps, ViewResp viewResp) { + public static List filterByDataSet(List metricResps, DataSetResp dataSetResp) { return metricResps.stream().filter(metricResp -> - viewResp.getAllMetrics().contains(metricResp.getId()) - || viewResp.getAllIncludeAllModels().contains(metricResp.getModelId())) + dataSetResp.getAllMetrics().contains(metricResp.getId()) + || dataSetResp.getAllIncludeAllModels().contains(metricResp.getModelId())) .collect(Collectors.toList()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 029203678..b28b6759a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -16,7 +16,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.MetricType; -import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; @@ -27,7 +26,17 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -37,14 +46,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; @Component @Slf4j @@ -103,7 +104,7 @@ public class QueryReqConverter { List tables = new ArrayList<>(); tables.add(metricTable); //4.build ParseSqlReq - ViewQueryParam result = new ViewQueryParam(); + DataSetQueryParam result = new DataSetQueryParam(); BeanUtils.copyProperties(querySQLReq, result); result.setTables(tables); @@ -118,17 +119,17 @@ public class QueryReqConverter { //6.physicalSql by ParseSqlReq queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); - queryStructReq.setViewId(querySQLReq.getViewId()); + queryStructReq.setDataSetId(querySQLReq.getDataSetId()); queryStructReq.setQueryType(getQueryType(aggOption)); log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); QueryParam queryParam = new QueryParam(); convert(queryStructReq, queryParam); QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryParam(queryParam); - queryStatement.setViewQueryParam(result); + queryStatement.setDataSetQueryParam(result); queryStatement.setIsS2SQL(true); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - queryStatement.setViewId(querySQLReq.getViewId()); + queryStatement.setDataSetId(querySQLReq.getDataSetId()); queryStatement.setEnableLimitWrapper(limitWrapper); return queryStatement; @@ -225,7 +226,7 @@ public class QueryReqConverter { public void correctTableName(QuerySqlReq querySqlReq) { String sql = querySqlReq.getSql(); sql = SqlReplaceHelper.replaceTable(sql, - Constants.TABLE_PREFIX + querySqlReq.getViewId()); + Constants.TABLE_PREFIX + querySqlReq.getDataSetId()); querySqlReq.setSql(sql); } @@ -239,7 +240,7 @@ public class QueryReqConverter { } private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, - ViewQueryParam viewQueryParam) { + DataSetQueryParam viewQueryParam) { String sql = viewQueryParam.getSql(); for (MetricTable metricTable : viewQueryParam.getTables()) { List measures = new ArrayList<>(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 2da3b9ccf..162f0c0d0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -77,7 +77,7 @@ public class QueryStructUtils { private List getDimensionIds(QueryStructReq queryStructReq) { List dimensionIds = new ArrayList<>(); MetaFilter metaFilter = new MetaFilter(); - metaFilter.setViewId(queryStructReq.getViewId()); + metaFilter.setDataSetId(queryStructReq.getDataSetId()); List dimensions = catalog.getDimensions(metaFilter); Map> pair = dimensions.stream() .collect(Collectors.groupingBy(DimensionResp::getBizName)); @@ -99,7 +99,7 @@ public class QueryStructUtils { private List getMetricIds(QueryStructReq queryStructCmd) { List metricIds = new ArrayList<>(); MetaFilter metaFilter = new MetaFilter(); - metaFilter.setViewId(queryStructCmd.getViewId()); + metaFilter.setDataSetId(queryStructCmd.getDataSetId()); List metrics = catalog.getMetrics(metaFilter); Map> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName)); for (Aggregator agg : queryStructCmd.getAggregators()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index 25399c801..b77e7de39 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -20,18 +20,18 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; -import com.tencent.supersonic.headless.server.service.ModelService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; @Component @@ -41,17 +41,13 @@ public class StatUtils { private static final TransmittableThreadLocal STATS = new TransmittableThreadLocal<>(); private final StatRepository statRepository; private final SqlFilterUtils sqlFilterUtils; - - private final ModelService modelService; private final ObjectMapper objectMapper = new ObjectMapper(); public StatUtils(StatRepository statRepository, - SqlFilterUtils sqlFilterUtils, - ModelService modelService) { + SqlFilterUtils sqlFilterUtils) { this.statRepository = statRepository; this.sqlFilterUtils = sqlFilterUtils; - this.modelService = modelService; } public static QueryStat get() { @@ -112,7 +108,7 @@ public class StatUtils { try { queryStatInfo.setTraceId(traceId) - .setViewId(queryTagReq.getViewId()) + .setDataSetId(queryTagReq.getDataSetId()) .setUser(user) .setQueryType(QueryType.STRUCT.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) @@ -150,7 +146,7 @@ public class StatUtils { try { queryStatInfo.setTraceId("") .setUser(userName) - .setViewId(querySqlReq.getViewId()) + .setDataSetId(querySqlReq.getDataSetId()) .setQueryType(QueryType.SQL.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) .setQuerySqlCmd(querySqlReq.toString()) @@ -180,7 +176,7 @@ public class StatUtils { try { queryStatInfo.setTraceId(traceId) - .setViewId(queryStructReq.getViewId()) + .setDataSetId(queryStructReq.getDataSetId()) .setUser(user) .setQueryType(QueryType.STRUCT.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java index 6fb8070a7..d0c667285 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java @@ -12,11 +12,8 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; +import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -24,6 +21,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + @Component @Slf4j public class TagReqConverter { @@ -61,7 +62,7 @@ public class TagReqConverter { List tables = new ArrayList<>(); tables.add(metricTable); //.build ParseSqlReq - ViewQueryParam result = new ViewQueryParam(); + DataSetQueryParam result = new DataSetQueryParam(); BeanUtils.copyProperties(querySqlReq, result); result.setTables(tables); DatabaseResp database = semanticSchemaResp.getDatabaseResp(); @@ -72,15 +73,15 @@ public class TagReqConverter { } //.physicalSql by ParseSqlReq queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql())); - queryStructReq.setViewId(querySqlReq.getViewId()); + queryStructReq.setDataSetId(querySqlReq.getDataSetId()); queryStructReq.setQueryType(QueryType.TAG); QueryParam queryParam = new QueryParam(); convert(queryTagReq, queryParam); queryStatement.setQueryParam(queryParam); - queryStatement.setViewQueryParam(result); + queryStatement.setDataSetQueryParam(result); queryStatement.setIsS2SQL(true); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - queryStatement.setViewId(queryTagReq.getViewId()); + queryStatement.setDataSetId(queryTagReq.getDataSetId()); queryStatement.setEnableLimitWrapper(limitWrapper); } return queryStatement; diff --git a/headless/server/src/main/resources/mapper/StatMapper.xml b/headless/server/src/main/resources/mapper/StatMapper.xml index e1b4d1a90..5de472000 100644 --- a/headless/server/src/main/resources/mapper/StatMapper.xml +++ b/headless/server/src/main/resources/mapper/StatMapper.xml @@ -9,7 +9,7 @@ - + @@ -43,13 +43,13 @@ insert into s2_query_stat_info ( - trace_id, model_id, view_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine, + trace_id, model_id, data_set_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine, elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols, order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key, query_opt_mode ) values ( - #{traceId}, #{modelId}, #{viewId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine}, + #{traceId}, #{modelId}, #{dataSetId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine}, #{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols}, #{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}, #{queryOptMode} ) @@ -61,7 +61,7 @@ from s2_query_stat_info - and start_time >= #{startTime} + and start_time >= #{startTime}String.valueOf(queryFilter.getValue()) and model_id = #{modelId} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index 47e8ea970..c1a5cf9df 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -23,7 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl; -import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl; +import com.tencent.supersonic.headless.server.service.impl.DataSetImpl; import com.tencent.supersonic.headless.server.utils.MetricConverter; import java.util.HashMap; import org.junit.jupiter.api.Assertions; @@ -66,7 +66,7 @@ public class MetricServiceImplTest { ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class); CollectService collectService = Mockito.mock(CollectService.class); ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class); - ViewService viewService = Mockito.mock(ViewServiceImpl.class); + DataSetService viewService = Mockito.mock(DataSetImpl.class); DimensionService dimensionService = Mockito.mock(DimensionService.class); return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, viewService, eventPublisher, dimensionService); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java index 87f15489e..b9e76b9bd 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java @@ -75,7 +75,7 @@ class ModelServiceImplTest { DomainService domainService = Mockito.mock(DomainService.class); UserService userService = Mockito.mock(UserService.class); DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); - ViewService viewService = Mockito.mock(ViewService.class); + DataSetService viewService = Mockito.mock(DataSetService.class); return new ModelServiceImpl(modelRepository, databaseService, dimensionService, metricService, domainService, userService, viewService, dateInfoRepository); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java index 76f38b21d..72720e303 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java @@ -13,8 +13,8 @@ import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.api.pojo.ViewDetail; -import com.tencent.supersonic.headless.api.pojo.ViewModelConfig; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.QueryConfig; @@ -24,11 +24,11 @@ import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -52,7 +52,7 @@ public class BenchMarkDemoDataLoader { private ModelRelaService modelRelaService; @Autowired - private ViewService viewService; + private DataSetService viewService; public void doRun() { try { @@ -214,22 +214,22 @@ public class BenchMarkDemoDataLoader { } public void addView_1() { - ViewReq viewReq = new ViewReq(); + DataSetReq viewReq = new DataSetReq(); viewReq.setName("cspider"); viewReq.setBizName("singer"); viewReq.setDomainId(3L); viewReq.setDescription("包含cspider数据集相关标签和指标信息"); viewReq.setAdmins(Lists.newArrayList("admin")); - List viewModelConfigs = Lists.newArrayList( - new ViewModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()), - new ViewModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()), - new ViewModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()), - new ViewModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L)) + List viewModelConfigs = Lists.newArrayList( + new DataSetModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()), + new DataSetModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()), + new DataSetModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()), + new DataSetModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L)) ); - ViewDetail viewDetail = new ViewDetail(); - viewDetail.setViewModelConfigs(viewModelConfigs); - viewReq.setViewDetail(viewDetail); - viewReq.setTypeEnum(TypeEnums.VIEW); + DataSetDetail viewDetail = new DataSetDetail(); + viewDetail.setDataSetModelConfigs(viewModelConfigs); + viewReq.setDataSetDetail(viewDetail); + viewReq.setTypeEnum(TypeEnums.DATASET); QueryConfig queryConfig = new QueryConfig(); TagTypeDefaultConfig tagTypeDefaultConfig = new TagTypeDefaultConfig(); TimeDefaultConfig tagTimeDefaultConfig = new TimeDefaultConfig(); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index d4861b53d..02e6108f1 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -134,7 +134,7 @@ public class ChatDemoLoader implements CommandLineRunner { private void addPlugin_1() { Plugin plugin1 = new Plugin(); plugin1.setType("WEB_PAGE"); - plugin1.setViewList(Arrays.asList(1L)); + plugin1.setDataSetList(Arrays.asList(1L)); plugin1.setPattern("用于分析超音数的流量概况，包含UV、PV等核心指标的追踪。P.S. 仅作为示例展示，无实际看板"); plugin1.setName("超音数流量分析看板"); PluginParseConfig pluginParseConfig = new PluginParseConfig(); @@ -169,14 +169,14 @@ public class ChatDemoLoader implements CommandLineRunner { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); ruleQueryTool.setId("0"); - ruleQueryTool.setViewIds(Lists.newArrayList(1L)); + ruleQueryTool.setDataSetIds(Lists.newArrayList(1L)); ruleQueryTool.setQueryTypes(Lists.newArrayList(QueryType.METRIC.name())); agentConfig.getTools().add(ruleQueryTool); if (demoEnabledNl2SqlLlm) { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setViewIds(Lists.newArrayList(-1L)); + llmParserTool.setDataSetIds(Lists.newArrayList(-1L)); agentConfig.getTools().add(llmParserTool); } agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); @@ -195,7 +195,7 @@ public class ChatDemoLoader implements CommandLineRunner { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setId("0"); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); - ruleQueryTool.setViewIds(Lists.newArrayList(2L)); + ruleQueryTool.setDataSetIds(Lists.newArrayList(2L)); ruleQueryTool.setQueryTypes(Lists.newArrayList(QueryType.TAG.name())); agentConfig.getTools().add(ruleQueryTool); @@ -203,7 +203,7 @@ public class ChatDemoLoader implements CommandLineRunner { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setViewIds(Lists.newArrayList(-1L)); + llmParserTool.setDataSetIds(Lists.newArrayList(-1L)); agentConfig.getTools().add(llmParserTool); } agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); @@ -225,7 +225,7 @@ public class ChatDemoLoader implements CommandLineRunner { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setViewIds(Lists.newArrayList(3L)); + llmParserTool.setDataSetIds(Lists.newArrayList(3L)); agentConfig.getTools().add(llmParserTool); } @@ -247,7 +247,7 @@ public class ChatDemoLoader implements CommandLineRunner { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setViewIds(Lists.newArrayList(4L)); + llmParserTool.setDataSetIds(Lists.newArrayList(4L)); agentConfig.getTools().add(llmParserTool); } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java index 44e796fe4..ea378fb09 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java @@ -15,21 +15,21 @@ import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.ViewDetail; -import com.tencent.supersonic.headless.api.pojo.ViewModelConfig; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -56,7 +56,7 @@ public class DuSQLDemoDataLoader { private MetricService metricService; @Autowired - private ViewService viewService; + private DataSetService viewService; public void doRun() { try { @@ -255,22 +255,22 @@ public class DuSQLDemoDataLoader { } public void addView_1() { - ViewReq viewReq = new ViewReq(); + DataSetReq viewReq = new DataSetReq(); viewReq.setName("DuSQL 互联网企业"); viewReq.setBizName("internet"); viewReq.setDomainId(4L); viewReq.setDescription("DuSQL互联网企业数据源相关的指标和维度等"); viewReq.setAdmins(Lists.newArrayList("admin")); - List viewModelConfigs = Lists.newArrayList( - new ViewModelConfig(9L, Lists.newArrayList(16L, 17L, 18L, 19L, 20L), Lists.newArrayList(10L, 11L)), - new ViewModelConfig(10L, Lists.newArrayList(21L, 22L, 23L), Lists.newArrayList(12L)), - new ViewModelConfig(11L, Lists.newArrayList(), Lists.newArrayList(13L, 14L, 15L)), - new ViewModelConfig(12L, Lists.newArrayList(24L), Lists.newArrayList(16L, 17L, 18L, 19L))); + List viewModelConfigs = Lists.newArrayList( + new DataSetModelConfig(9L, Lists.newArrayList(16L, 17L, 18L, 19L, 20L), Lists.newArrayList(10L, 11L)), + new DataSetModelConfig(10L, Lists.newArrayList(21L, 22L, 23L), Lists.newArrayList(12L)), + new DataSetModelConfig(11L, Lists.newArrayList(), Lists.newArrayList(13L, 14L, 15L)), + new DataSetModelConfig(12L, Lists.newArrayList(24L), Lists.newArrayList(16L, 17L, 18L, 19L))); - ViewDetail viewDetail = new ViewDetail(); - viewDetail.setViewModelConfigs(viewModelConfigs); - viewReq.setViewDetail(viewDetail); - viewReq.setTypeEnum(TypeEnums.VIEW); + DataSetDetail viewDetail = new DataSetDetail(); + viewDetail.setDataSetModelConfigs(viewModelConfigs); + viewReq.setDataSetDetail(viewDetail); + viewReq.setTypeEnum(TypeEnums.DATASET); QueryConfig queryConfig = new QueryConfig(); MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index e9b107872..efbb0c111 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -38,21 +38,21 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.RelateDimension; -import com.tencent.supersonic.headless.api.pojo.ViewDetail; -import com.tencent.supersonic.headless.api.pojo.ViewModelConfig; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -84,7 +84,7 @@ public class ModelDemoDataLoader { @Autowired private AuthService authService; @Autowired - private ViewService viewService; + private DataSetService dataSetService; @Autowired private DataSourceProperties dataSourceProperties; @@ -446,21 +446,21 @@ public class ModelDemoDataLoader { } public void addView_1() { - ViewReq viewReq = new ViewReq(); - viewReq.setName("超音数"); - viewReq.setBizName("s2"); - viewReq.setDomainId(1L); - viewReq.setDescription("包含超音数访问统计相关的指标和维度等"); - viewReq.setAdmins(Lists.newArrayList("admin")); - List viewModelConfigs = Lists.newArrayList( - new ViewModelConfig(1L, Lists.newArrayList(1L, 2L), Lists.newArrayList()), - new ViewModelConfig(2L, Lists.newArrayList(), Lists.newArrayList(1L, 2L, 3L)), - new ViewModelConfig(3L, Lists.newArrayList(3L), Lists.newArrayList(4L))); + DataSetReq dataSetReq = new DataSetReq(); + dataSetReq.setName("超音数"); + dataSetReq.setBizName("s2"); + dataSetReq.setDomainId(1L); + dataSetReq.setDescription("包含超音数访问统计相关的指标和维度等"); + dataSetReq.setAdmins(Lists.newArrayList("admin")); + List dataSetModelConfigs = Lists.newArrayList( + new DataSetModelConfig(1L, Lists.newArrayList(1L, 2L), Lists.newArrayList()), + new DataSetModelConfig(2L, Lists.newArrayList(), Lists.newArrayList(1L, 2L, 3L)), + new DataSetModelConfig(3L, Lists.newArrayList(3L), Lists.newArrayList(4L))); - ViewDetail viewDetail = new ViewDetail(); - viewDetail.setViewModelConfigs(viewModelConfigs); - viewReq.setViewDetail(viewDetail); - viewReq.setTypeEnum(TypeEnums.VIEW); + DataSetDetail dataSetDetail = new DataSetDetail(); + dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs); + dataSetReq.setDataSetDetail(dataSetDetail); + dataSetReq.setTypeEnum(TypeEnums.DATASET); QueryConfig queryConfig = new QueryConfig(); MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); @@ -468,25 +468,25 @@ public class ModelDemoDataLoader { timeDefaultConfig.setUnit(7); metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig); queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig); - viewReq.setQueryConfig(queryConfig); - viewService.save(viewReq, User.getFakeUser()); + dataSetReq.setQueryConfig(queryConfig); + dataSetService.save(dataSetReq, User.getFakeUser()); } public void addView_2() { - ViewReq viewReq = new ViewReq(); - viewReq.setName("艺人库"); - viewReq.setBizName("singer"); - viewReq.setDomainId(2L); - viewReq.setDescription("包含艺人相关标签和指标信息"); - viewReq.setAdmins(Lists.newArrayList("admin", "jack")); - List viewModelConfigs = Lists.newArrayList( - new ViewModelConfig(4L, Lists.newArrayList(4L, 5L, 6L, 7L), + DataSetReq dataSetReq = new DataSetReq(); + dataSetReq.setName("艺人库"); + dataSetReq.setBizName("singer"); + dataSetReq.setDomainId(2L); + dataSetReq.setDescription("包含艺人相关标签和指标信息"); + dataSetReq.setAdmins(Lists.newArrayList("admin", "jack")); + List dataSetModelConfigs = Lists.newArrayList( + new DataSetModelConfig(4L, Lists.newArrayList(4L, 5L, 6L, 7L), Lists.newArrayList(5L, 6L, 7L)) ); - ViewDetail viewDetail = new ViewDetail(); - viewDetail.setViewModelConfigs(viewModelConfigs); - viewReq.setViewDetail(viewDetail); - viewReq.setTypeEnum(TypeEnums.VIEW); + DataSetDetail dataSetDetail = new DataSetDetail(); + dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs); + dataSetReq.setDataSetDetail(dataSetDetail); + dataSetReq.setTypeEnum(TypeEnums.DATASET); QueryConfig queryConfig = new QueryConfig(); TagTypeDefaultConfig tagTypeDefaultConfig = new TagTypeDefaultConfig(); TimeDefaultConfig tagTimeDefaultConfig = new TimeDefaultConfig(); @@ -504,8 +504,8 @@ public class ModelDemoDataLoader { metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig); queryConfig.setTagTypeDefaultConfig(tagTypeDefaultConfig); queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig); - viewReq.setQueryConfig(queryConfig); - viewService.save(viewReq, User.getFakeUser()); + dataSetReq.setQueryConfig(queryConfig); + dataSetService.save(dataSetReq, User.getFakeUser()); } public void addAuthGroup_1() { diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index c1721f2be..9148f838f 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -28,8 +28,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\ com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter -com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ - com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver +com.tencent.supersonic.chat.core.parser.sql.llm.DataSetResolver=\ + com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicDataSetResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index 9d85c8e50..08d31254f 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -242,3 +242,10 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; + + +--20240229 +alter table s2_view rename to s2_data_set; +alter table s2_query_stat_info change view_id data_set_id bigint; +alter table s2_plugin change `view` data_set varchar(200); +alter table s2_data_set change view_detail data_set_detail text; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/data-h2.sql b/launchers/standalone/src/main/resources/db/data-h2.sql index 95b86d684..a3321f732 100644 --- a/launchers/standalone/src/main/resources/db/data-h2.sql +++ b/launchers/standalone/src/main/resources/db/data-h2.sql @@ -1112,17 +1112,17 @@ MERGE INTO song(imp_date,song_name,artist_name,country,f_id,g_name,rating,langua MERGE INTO song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'阿杰伊阿卡什','Topu','印度',6,'现代',10,'孟加拉语','27-MAR-2004',320); -insert into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_131','百度集团','北京','2000','李彦宏','李彦宏',102300000000,40000); -insert into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_132','阿里巴巴集团','杭州','1999年','马云','张勇',376800000000,103699); -insert into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_133','深圳市腾讯计算机系统有限公司','深圳','1998','马化腾','刘炽平',321600000000,56310); -insert into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_134','北京京东世纪贸易有限公司','北京','1998','刘强东','刘强东',28800000000,179000); -insert into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_135','网易公司','杭州','1997','丁磊','丁磊',67500000000,20000); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_131','百度集团','北京','2000','李彦宏','李彦宏',102300000000,40000); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_132','阿里巴巴集团','杭州','1999年','马云','张勇',376800000000,103699); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_133','深圳市腾讯计算机系统有限公司','深圳','1998','马化腾','刘炽平',321600000000,56310); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_134','北京京东世纪贸易有限公司','北京','1998','刘强东','刘强东',28800000000,179000); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_135','网易公司','杭州','1997','丁磊','丁磊',67500000000,20000); -insert into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_136','阿里云','2009年9月10日','item_enterprise_13_132','张勇',50000000); -insert into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_137','天猫','2012年1月11日','item_enterprise_13_132','张勇',100000000); -insert into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_138','腾讯游戏','2003','item_enterprise_13_133','马化腾',50000000); -insert into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_139','度小满','2018','item_enterprise_13_131','朱光',100000000); -insert into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_140','京东金融','2017','item_enterprise_13_134','刘强东',100000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_136','阿里云','2009年9月10日','item_enterprise_13_132','张勇',50000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_137','天猫','2012年1月11日','item_enterprise_13_132','张勇',100000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_138','腾讯游戏','2003','item_enterprise_13_133','马化腾',50000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_139','度小满','2018','item_enterprise_13_131','朱光',100000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_140','京东金融','2017','item_enterprise_13_134','刘强东',100000000); insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_131','item_enterprise_13_139',10,10,30); insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_133','item_enterprise_13_138',80,80,60); diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 21a4154f0..b0aa99435 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -248,7 +248,7 @@ CREATE TABLE IF NOT EXISTS `s2_query_stat_info` ( `id` INT NOT NULL AUTO_INCREMENT, `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier `model_id` INT DEFAULT NULL, - `view_id` INT DEFAULT NULL, + `data_set_id` INT DEFAULT NULL, `user` varchar(200) DEFAULT NULL, `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene @@ -326,7 +326,7 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` ( `id` INT AUTO_INCREMENT, `type` varchar(50) NULL, - `view` varchar(100) NULL, + `data_set` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, `parse_mode_config` LONGVARCHAR NULL, @@ -556,7 +556,7 @@ CREATE TABLE IF NOT EXISTS `s2_app` ( updated_by VARCHAR(255) ); -CREATE TABLE IF NOT EXISTS `s2_view` ( +CREATE TABLE IF NOT EXISTS `s2_data_set` ( id BIGINT AUTO_INCREMENT PRIMARY KEY, domain_id BIGINT, `name` VARCHAR(255), @@ -564,7 +564,7 @@ CREATE TABLE IF NOT EXISTS `s2_view` ( description VARCHAR(255), status INT, alias VARCHAR(255), - view_detail TEXT, + data_set_detail TEXT, created_at TIMESTAMP, created_by VARCHAR(255), updated_at TIMESTAMP, diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java index 94b134de8..6e0d71294 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java @@ -112,7 +112,7 @@ public class QueryBySqlTest extends BaseTest { SemanticQueryResp semanticQueryResp = queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom); Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage()); - Assertions.assertTrue(semanticQueryResp.getSql().contains("`user_name` = 'tom'")); + Assertions.assertTrue(semanticQueryResp.getSql().contains("user_name = 'tom'")); } } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java index 40a9b12f3..2393cbd80 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java @@ -5,10 +5,10 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -22,7 +22,7 @@ public class SchemaAuthTest extends BaseTest { private DomainService domainService; @Autowired - private ViewService viewService; + private DataSetService viewService; @Autowired private ModelService modelService; @@ -57,10 +57,10 @@ public class SchemaAuthTest extends BaseTest { @Test public void test_getViewList_alice() { User user = DataUtils.getUserAlice(); - List modelResps = viewService.getViewsInheritAuth(user, 0L); + List modelResps = viewService.getDataSetsInheritAuth(user, 0L); List expectedViewIds = Lists.newArrayList(2L); Assertions.assertEquals(expectedViewIds, - modelResps.stream().map(ViewResp::getId).collect(Collectors.toList())); + modelResps.stream().map(DataSetResp::getId).collect(Collectors.toList())); } @Test @@ -84,10 +84,10 @@ public class SchemaAuthTest extends BaseTest { @Test public void test_getViewList_jack() { User user = DataUtils.getUserJack(); - List modelResps = viewService.getViewsInheritAuth(user, 0L); + List modelResps = viewService.getDataSetsInheritAuth(user, 0L); List expectedViewIds = Lists.newArrayList(1L, 2L); Assertions.assertEquals(expectedViewIds, - modelResps.stream().map(ViewResp::getId).collect(Collectors.toList())); + modelResps.stream().map(DataSetResp::getId).collect(Collectors.toList())); } } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 766e84531..4e269b69f 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -138,7 +138,7 @@ public class DataUtils { private static RuleParserTool getRuleQueryTool() { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); - ruleQueryTool.setViewIds(Lists.newArrayList(-1L)); + ruleQueryTool.setDataSetIds(Lists.newArrayList(-1L)); ruleQueryTool.setQueryModes(Lists.newArrayList("METRIC_TAG", "METRIC_FILTER", "METRIC_MODEL", "TAG_DETAIL", "TAG_LIST_FILTER", "TAG_ID")); return ruleQueryTool; diff --git a/launchers/standalone/src/test/resources/application-local.yaml b/launchers/standalone/src/test/resources/application-local.yaml index d30eec7c3..1fb4f1692 100644 --- a/launchers/standalone/src/test/resources/application-local.yaml +++ b/launchers/standalone/src/test/resources/application-local.yaml @@ -77,4 +77,4 @@ logging: inMemoryEmbeddingStore: persistent: - path: d:// + path: /tmp diff --git a/launchers/standalone/src/test/resources/db/data-h2.sql b/launchers/standalone/src/test/resources/db/data-h2.sql index 860fb7264..a3321f732 100644 --- a/launchers/standalone/src/test/resources/db/data-h2.sql +++ b/launchers/standalone/src/test/resources/db/data-h2.sql @@ -5,7 +5,6 @@ MERGE INTO s2_user (id, `name`, password, display_name, email) values (3, 'tom', MERGE INTO s2_user (id, `name`, password, display_name, email, is_admin) values (4, 'lucy','123456','lucy','lucy@xx.com', 1); MERGE INTO s2_user (id, `name`, password, display_name, email) values (5, 'alice','123456','alice','alice@xx.com'); - MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (1 , 1, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin'); MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) @@ -13,7 +12,6 @@ values (2 , 2, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), D MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (3 , 3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin'); - MERGE INTO s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin'); @@ -1113,4 +1111,29 @@ MERGE INTO song(imp_date,song_name,artist_name,country,f_id,g_name,rating,langua MERGE INTO song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'打败它','Michel','英国',5,'流行',8,'英文','17-MAR-2002',720); MERGE INTO song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'阿杰伊阿卡什','Topu','印度',6,'现代',10,'孟加拉语','27-MAR-2004',320); + +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_131','百度集团','北京','2000','李彦宏','李彦宏',102300000000,40000); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_132','阿里巴巴集团','杭州','1999年','马云','张勇',376800000000,103699); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_133','深圳市腾讯计算机系统有限公司','深圳','1998','马化腾','刘炽平',321600000000,56310); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_134','北京京东世纪贸易有限公司','北京','1998','刘强东','刘强东',28800000000,179000); +MERGE into company(imp_date,company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_135','网易公司','杭州','1997','丁磊','丁磊',67500000000,20000); + +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_136','阿里云','2009年9月10日','item_enterprise_13_132','张勇',50000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_137','天猫','2012年1月11日','item_enterprise_13_132','张勇',100000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_138','腾讯游戏','2003','item_enterprise_13_133','马化腾',50000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_139','度小满','2018','item_enterprise_13_131','朱光',100000000); +MERGE into brand(imp_date,brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES (DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_140','京东金融','2017','item_enterprise_13_134','刘强东',100000000); + +insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_131','item_enterprise_13_139',10,10,30); +insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_133','item_enterprise_13_138',80,80,60); +insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_134','item_enterprise_13_140',80,80,60); +insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_132','item_enterprise_13_137',80,80,60); +insert into company_revenue(imp_date,company_id,brand_id,revenue_proportion,profit_proportion,expenditure_proportion) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'item_enterprise_13_132','item_enterprise_13_136',10,10,30); + +insert into company_brand_revenue(imp_date,year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES (DATEADD('DAY', -1, CURRENT_DATE()), '2018','item_enterprise_13_138',500000000,-300000000,10,-10); +insert into company_brand_revenue(imp_date,year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES (DATEADD('DAY', -1, CURRENT_DATE()), '2019','item_enterprise_13_136',100000000000,50000000000,100,50); +insert into company_brand_revenue(imp_date,year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'2018','item_enterprise_13_137',100000000000,50000000000,100,-10); +insert into company_brand_revenue(imp_date,year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES (DATEADD('DAY', -1, CURRENT_DATE()), '2018','item_enterprise_13_139',500000000,50000000000,10,50); +insert into company_brand_revenue(imp_date,year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES ( DATEADD('DAY', -1, CURRENT_DATE()),'2018','item_enterprise_13_140',100000000000,-300000000,10,50); + -- benchmark diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index e12dd1d64..e8fc30dba 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -242,14 +242,14 @@ create table IF NOT EXISTS s2_canvas updated_by varchar(100) not null, PRIMARY KEY (`id`) ); -COMMENT ON TABLE s2_canvas IS 'view information table'; +COMMENT ON TABLE s2_canvas IS 'canvas table'; CREATE TABLE IF NOT EXISTS `s2_query_stat_info` ( `id` INT NOT NULL AUTO_INCREMENT, `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier `model_id` INT DEFAULT NULL, - `view_id` INT DEFAULT NULL, + `data_set_id` INT DEFAULT NULL, `user` varchar(200) DEFAULT NULL, `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene @@ -327,7 +327,7 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` ( `id` INT AUTO_INCREMENT, `type` varchar(50) NULL, - `view` varchar(100) NULL, + `data_set` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, `parse_mode_config` LONGVARCHAR NULL, @@ -467,6 +467,51 @@ CREATE TABLE IF NOT EXISTS `song` ( ); COMMENT ON TABLE song IS 'song'; +CREATE TABLE IF NOT EXISTS `company` ( + `imp_date` varchar(50) , + `company_id` varchar(50) NOT NULL , + `company_name` varchar(50) NOT NULL , + `headquarter_address` varchar(50) NOT NULL , + `company_established_time` varchar(20) NOT NULL , + `founder` varchar(20) NOT NULL , + `ceo` varchar(20) NOT NULL , + `annual_turnover` bigint(15) , + `employee_count` int(7) , + PRIMARY KEY (`company_id`) + ); + +CREATE TABLE IF NOT EXISTS `brand` ( + `imp_date` varchar(50) , + `brand_id` varchar(50) NOT NULL , + `brand_name` varchar(50) NOT NULL , + `brand_established_time` varchar(20) NOT NULL , + `company_id` varchar(50) NOT NULL , + `legal_representative` varchar(20) NOT NULL , + `registered_capital` bigint(15) , + PRIMARY KEY (`brand_id`) + ); + +CREATE TABLE IF NOT EXISTS `company_revenue` ( + `imp_date` varchar(50) , + `company_id` varchar(50) NOT NULL , + `brand_id` varchar(50) NOT NULL , + `revenue_proportion` double NOT NULL, + `profit_proportion` double NOT NULL , + `expenditure_proportion` double NOT NULL + ); + +CREATE TABLE IF NOT EXISTS `company_brand_revenue` ( + `imp_date` varchar(50) , + `year_time` varchar(10) NOT NULL , + `brand_id` varchar(50) NOT NULL , + `revenue` bigint(15) NOT NULL, + `profit` bigint(15) NOT NULL , + `revenue_growth_year_on_year` double NOT NULL , + `profit_growth_year_on_year` double NOT NULL + ); + + + CREATE TABLE IF NOT EXISTS s2_sys_parameter ( id INT PRIMARY KEY AUTO_INCREMENT, @@ -512,7 +557,7 @@ CREATE TABLE IF NOT EXISTS `s2_app` ( updated_by VARCHAR(255) ); -CREATE TABLE IF NOT EXISTS `s2_view` ( +CREATE TABLE IF NOT EXISTS `s2_data_set` ( id BIGINT AUTO_INCREMENT PRIMARY KEY, domain_id BIGINT, `name` VARCHAR(255), @@ -520,12 +565,11 @@ CREATE TABLE IF NOT EXISTS `s2_view` ( description VARCHAR(255), status INT, alias VARCHAR(255), - view_detail TEXT, + data_set_detail TEXT, created_at TIMESTAMP, created_by VARCHAR(255), updated_at TIMESTAMP, updated_by VARCHAR(255), - filter_sql VARCHAR(1000), query_config VARCHAR(3000), `admin` varchar(3000) DEFAULT NULL, `admin_org` varchar(3000) DEFAULT NULL