diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java index 0374f10a3..36da61d57 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.chat.server.agent; import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; import com.tencent.supersonic.chat.server.memory.MemoryReviewTask; import com.tencent.supersonic.common.pojo.ChatApp; import com.tencent.supersonic.common.pojo.RecordInfo; +import com.tencent.supersonic.common.pojo.User; import lombok.Data; import org.springframework.util.CollectionUtils; @@ -12,6 +14,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; @Data @@ -33,6 +36,8 @@ public class Agent extends RecordInfo { private String toolConfig; private Map chatAppConfig = Collections.emptyMap(); private VisualConfig visualConfig; + private List admins = Lists.newArrayList(); + private List viewers = Lists.newArrayList(); public List getTools(AgentToolType type) { Map map = JSONObject.parseObject(toolConfig, Map.class); @@ -105,4 +110,9 @@ public class Agent extends RecordInfo { .filter(dataSetIds -> !CollectionUtils.isEmpty(dataSetIds)) .flatMap(Collection::stream).collect(Collectors.toSet()); } + + public boolean contains(User user, Function> list) { + return list.apply(this).contains(user.getName()); + } + } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java index 58645c621..a71596e82 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java @@ -40,4 +40,8 @@ public class AgentDO { private String chatModelConfig; private String visualConfig; + + private String admin; + + private String viewer; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java index 0cb4ddee0..ff3e01b18 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.agent.AgentToolType; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.common.pojo.User; +import com.tencent.supersonic.common.pojo.enums.AuthType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -15,6 +16,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -48,8 +50,11 @@ public class AgentController { } @RequestMapping("/getAgentList") - public List getAgentList() { - return agentService.getAgents(); + public List getAgentList( + @RequestParam(value = "authType", required = false) AuthType authType, + HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { + User user = UserHolder.findUser(httpServletRequest, httpServletResponse); + return agentService.getAgents(user, authType); } @RequestMapping("/getToolTypes") diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java index 147ff2615..d5d24fcb7 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java @@ -2,10 +2,12 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.common.pojo.User; +import com.tencent.supersonic.common.pojo.enums.AuthType; import java.util.List; public interface AgentService { + List getAgents(User user, AuthType authType); List getAgents(); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index c048c59fc..591d3a21b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -14,6 +14,7 @@ import com.tencent.supersonic.chat.server.service.MemoryService; import com.tencent.supersonic.common.config.ChatModel; import com.tencent.supersonic.common.pojo.ChatApp; import com.tencent.supersonic.common.pojo.User; +import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.service.ChatModelService; import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; @@ -43,6 +44,27 @@ public class AgentServiceImpl extends ServiceImpl implem private ExecutorService executorService = Executors.newFixedThreadPool(1); + @Override + public List getAgents(User user, AuthType authType) { + return getAgentDOList().stream().map(this::convert) + .filter(agent -> filterByAuth(agent, user, authType)).collect(Collectors.toList()); + } + + private boolean filterByAuth(Agent agent, User user, AuthType authType) { + if (user.isSuperAdmin() || user.getName().equals(agent.getCreatedBy())) { + return true; + } + authType = authType == null ? AuthType.VIEWER : authType; + switch (authType) { + case ADMIN: + return agent.contains(user, Agent::getAdmins); + case VIEWER: + default: + return agent.contains(user, Agent::getAdmins) + || agent.contains(user, Agent::getViewers); + } + } + @Override public List getAgents() { return getAgentDOList().stream().map(this::convert).collect(Collectors.toList()); @@ -135,6 +157,8 @@ public class AgentServiceImpl extends ServiceImpl implem c.setChatModelConfig(chatModelService.getChatModel(c.getChatModelId()).getConfig()); } }); + agent.setAdmins(JsonUtil.toList(agentDO.getAdmin(), String.class)); + agent.setViewers(JsonUtil.toList(agentDO.getViewer(), String.class)); return agent; } @@ -145,6 +169,8 @@ public class AgentServiceImpl extends ServiceImpl implem agentDO.setExamples(JsonUtil.toString(agent.getExamples())); agentDO.setChatModelConfig(JsonUtil.toString(agent.getChatAppConfig())); agentDO.setVisualConfig(JsonUtil.toString(agent.getVisualConfig())); + agentDO.setAdmin(JsonUtil.toString(agent.getAdmins())); + agentDO.setViewer(JsonUtil.toString(agent.getViewers())); if (agentDO.getStatus() == null) { agentDO.setStatus(1); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java index 0c505a120..3df7e526b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java @@ -1,5 +1,5 @@ package com.tencent.supersonic.common.pojo.enums; public enum AuthType { - VISIBLE, ADMIN + VIEWER, ADMIN } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index fd7c39836..767b7dfdf 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -7,14 +7,11 @@ import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; import java.util.HashSet; import java.util.List; import java.util.Set; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; - @Data @AllArgsConstructor @NoArgsConstructor @@ -32,13 +29,6 @@ public class SemanticSchemaResp { private DatabaseResp databaseResp; private QueryType queryType; - public String getSchemaKey() { - if (dataSetId == null) { - return String.format("%s_%s", schemaType, StringUtils.join(modelIds, UNDERLINE)); - } - return String.format("%s_%s", schemaType, dataSetId); - } - public MetricSchemaResp getMetric(String bizName) { return metrics.stream().filter(metric -> bizName.equalsIgnoreCase(metric.getBizName())) .findFirst().orElse(null); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java index 5f8335cc8..a8ebb910a 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java @@ -36,8 +36,8 @@ public class LLMSqlCorrector extends BaseSemanticCorrector { + "\n2.NO NEED to check date filters as the junior engineer seldom makes mistakes in this regard." + "\n3.DO NOT miss the AGGREGATE operator of metrics, always add it as needed." + "\n4.ALWAYS use `with` statement if nested aggregation is needed." - + "\n5.ALWAYS enclose alias created by `AS` command in underscores." - + "\n6.ALWAYS translate alias created by `AS` command to the same language as the `#Question`." + + "\n5.ALWAYS enclose alias declared by `AS` command in underscores." + + "\n6.Alias created by `AS` command must be in the same language ast the `Question`." + "\n#Question:{{question}} #InputSQL:{{sql}} #Response:"; public LLMSqlCorrector() { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java index fd5f176e3..a41be00ab 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java @@ -36,15 +36,13 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy { + "\n#Task: You will be provided with a natural language question asked by users," + "please convert it to a SQL query so that relevant data could be returned " + "by executing the SQL query against underlying database." + "\n#Rules:" - + "\n1.ALWAYS generate columns and values specified in the `Schema`, DO NOT hallucinate." - + "\n2.ALWAYS be cautious, word in the `Schema` does not mean it must appear in the SQL." - + "\n3.ALWAYS specify date filter using `>`,`<`,`>=`,`<=` operator." - + "\n4.DO NOT include date filter in the where clause if not explicitly expressed in the `Question`." - + "\n5.DO NOT calculate date range using functions." - + "\n6.DO NOT miss the AGGREGATE operator of metrics, always add it as needed." - + "\n7.ALWAYS use `with` statement if nested aggregation is needed." - + "\n8.ALWAYS enclose alias created by `AS` command in underscores." - + "\n9.ALWAYS translate alias created by `AS` command to the same language as the `#Question`." + + "\n1.SQL columns and values must be mentioned in the `Schema`, DO NOT hallucinate." + + "\n2.ALWAYS specify date filter using `>`,`<`,`>=`,`<=` operator." + + "\n3.DO NOT include date filter in the where clause if not explicitly expressed in the `Question`." + + "\n4.DO NOT calculate date range using functions." + + "\n5.ALWAYS use `with` statement if nested aggregation is needed." + + "\n6.ALWAYS enclose alias declared by `AS` command in underscores." + + "\n7.Alias created by `AS` command must be in the same language ast the `Question`." + "\n#Exemplars: {{exemplar}}" + "\n#Query: Question:{{question}},Schema:{{schema}},SideInfo:{{information}}"; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java index 4ad32b274..48d191015 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java @@ -21,12 +21,6 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { super(); } - @Override - public List match(List candidateElementMatches, - ChatQueryContext queryCtx) { - return super.match(candidateElementMatches, queryCtx); - } - @Override public void fillParseInfo(ChatQueryContext chatQueryContext, Long dataSetId) { super.fillParseInfo(chatQueryContext, dataSetId); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java index be4767cab..a4d787b9e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java @@ -25,12 +25,6 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1); } - @Override - public List match(List candidateElementMatches, - ChatQueryContext queryCtx) { - return super.match(candidateElementMatches, queryCtx); - } - @Override public void fillParseInfo(ChatQueryContext chatQueryContext, Long dataSetId) { super.fillParseInfo(chatQueryContext, dataSetId); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java index fd26da4d2..282a60a35 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java @@ -4,9 +4,9 @@ import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.core.pojo.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.TimeRange; -import com.tencent.supersonic.headless.core.translator.calcite.schema.DataSourceTable; -import com.tencent.supersonic.headless.core.translator.calcite.schema.DataSourceTable.Builder; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteTable; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteTable.Builder; +import com.tencent.supersonic.headless.core.translator.calcite.sql.SchemaBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.config.CalciteConnectionConfigImpl; @@ -156,14 +156,14 @@ public abstract class AbstractAccelerator implements QueryAccelerator { String[] dbTable = materialization.getName().split("\\."); String tb = dbTable[1].toLowerCase(); String db = dbTable[0].toLowerCase(); - Builder builder = DataSourceTable.newBuilder(tb); + Builder builder = S2CalciteTable.newBuilder(tb); for (String f : materialization.getColumns()) { builder.addField(f, SqlTypeName.VARCHAR); } if (StringUtils.isNotBlank(materialization.getPartitionName())) { builder.addField(materialization.getPartitionName(), SqlTypeName.VARCHAR); } - DataSourceTable srcTable = builder.withRowCount(1L).build(); + S2CalciteTable srcTable = builder.withRowCount(1L).build(); if (Objects.nonNull(db) && !db.isEmpty()) { SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db); if (Objects.isNull(schemaPlus)) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index 6c4e8ed85..a218fa0fb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -38,7 +38,7 @@ public class JdbcExecutor implements QueryExecutor { SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class); String sql = StringUtils.normalizeSpace(queryStatement.getSql()); log.info("executing SQL: {}", sql); - Database database = queryStatement.getSemanticModel().getDatabase(); + Database database = queryStatement.getOntology().getDatabase(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); try { SqlUtils sqlUtil = sqlUtils.init(database); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 28e0c3c8d..33266cc7a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -29,7 +29,7 @@ public class QueryStatement { private String dataSetAlias; private String dataSetSimplifySql; private Boolean enableLimitWrapper = false; - private SemanticModel semanticModel; + private Ontology ontology; private SemanticSchemaResp semanticSchemaResp; private Integer limit = 1000; private Boolean isTranslated = false; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index 0f59f50b7..a1f7a5f48 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -1,34 +1,72 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; +import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; +import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.StringUtil; +import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.api.pojo.enums.MetricType; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; import com.tencent.supersonic.headless.core.utils.ComponentFactory; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; @Component @Slf4j public class DefaultSemanticTranslator implements SemanticTranslator { + @Autowired + private SqlGenerateUtils sqlGenerateUtils; + public void translate(QueryStatement queryStatement) { + if (queryStatement.isTranslated()) { + return; + } + try { + preprocess(queryStatement); parse(queryStatement); optimize(queryStatement); } catch (Exception e) { @@ -36,13 +74,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } } - public void optimize(QueryStatement queryStatement) { - for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { - queryOptimizer.rewrite(queryStatement); - } - } - - public void parse(QueryStatement queryStatement) throws Exception { + private void parse(QueryStatement queryStatement) throws Exception { QueryParam queryParam = queryStatement.getQueryParam(); if (Objects.isNull(queryStatement.getDataSetQueryParam())) { queryStatement.setDataSetQueryParam(new DataSetQueryParam()); @@ -50,6 +82,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator { if (Objects.isNull(queryStatement.getMetricQueryParam())) { queryStatement.setMetricQueryParam(new MetricQueryParam()); } + log.debug("SemanticConverter before [{}]", queryParam); for (QueryConverter headlessConverter : ComponentFactory.getQueryConverters()) { if (headlessConverter.accept(queryStatement)) { @@ -59,13 +92,16 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } log.debug("SemanticConverter after {} {} {}", queryParam, queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam()); + if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) { doParse(queryStatement.getDataSetQueryParam(), queryStatement); } else { queryStatement.getMetricQueryParam() .setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); - doParse(queryStatement); + doParse(queryStatement, + AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery())); } + if (StringUtils.isEmpty(queryStatement.getSql())) { throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg()); } @@ -77,11 +113,11 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } } - public QueryStatement doParse(DataSetQueryParam dataSetQueryParam, + private QueryStatement doParse(DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) { log.info("parse dataSetQuery [{}] ", dataSetQueryParam); - SemanticModel semanticModel = queryStatement.getSemanticModel(); - EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType()); + Ontology ontology = queryStatement.getOntology(); + EngineType engineType = EngineType.fromString(ontology.getDatabase().getType()); try { if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) { List tables = new ArrayList<>(); @@ -132,12 +168,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator { return queryStatement; } - public QueryStatement doParse(QueryStatement queryStatement) { - return doParse(queryStatement, - AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery())); - } - - public QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) { + private QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) { MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam(); log.info("parse metricQuery [{}] isAgg [{}]", metricQueryParam, isAgg); try { @@ -151,18 +182,19 @@ public class DefaultSemanticTranslator implements SemanticTranslator { private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) throws Exception { - MetricQueryParam metricReq = new MetricQueryParam(); - metricReq.setMetrics(metricTable.getMetrics()); - metricReq.setDimensions(metricTable.getDimensions()); - metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere())); - metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption())); + MetricQueryParam metricQueryParam = new MetricQueryParam(); + metricQueryParam.setMetrics(metricTable.getMetrics()); + metricQueryParam.setDimensions(metricTable.getDimensions()); + metricQueryParam.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere())); + metricQueryParam.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption())); + QueryStatement tableSql = new QueryStatement(); tableSql.setIsS2SQL(false); - tableSql.setMetricQueryParam(metricReq); + tableSql.setMetricQueryParam(metricQueryParam); tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); tableSql.setDataSetId(queryStatement.getDataSetId()); - tableSql.setSemanticModel(queryStatement.getSemanticModel()); + tableSql.setOntology(queryStatement.getOntology()); if (isSingleMetricTable) { tableSql.setDataSetSql(dataSetQueryParam.getSql()); tableSql.setDataSetAlias(metricTable.getAlias()); @@ -174,4 +206,302 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } return tableSql; } + + private void optimize(QueryStatement queryStatement) { + for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { + queryOptimizer.rewrite(queryStatement); + } + } + + private void preprocess(QueryStatement queryStatement) { + if (StringUtils.isBlank(queryStatement.getSql())) { + return; + } + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + + convertNameToBizName(queryStatement); + rewriteFunction(queryStatement); + queryStatement.setSql(SqlRemoveHelper.removeUnderscores(queryStatement.getSql())); + + String tableName = SqlSelectHelper.getTableName(queryStatement.getSql()); + if (StringUtils.isEmpty(tableName)) { + return; + } + // correct order item is same as agg alias + String reqSql = queryStatement.getSql(); + queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderItem(queryStatement.getSql())); + log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql()); + // 5.build MetricTables + List allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql()); + List metricSchemas = getMetrics(semanticSchemaResp, allFields); + List metrics = + metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList()); + Set dimensions = getDimensions(semanticSchemaResp, allFields); + QueryStructReq queryStructReq = new QueryStructReq(); + + MetricTable metricTable = new MetricTable(); + metricTable.getMetrics().addAll(metrics); + metricTable.getDimensions().addAll(dimensions); + metricTable.setAlias(tableName.toLowerCase()); + // if metric empty , fill model default + if (CollectionUtils.isEmpty(metricTable.getMetrics())) { + metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( + getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); + } else { + queryStructReq.getAggregators() + .addAll(metricTable.getMetrics().stream() + .map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)) + .collect(Collectors.toList())); + } + AggOption aggOption = getAggOption(queryStatement, metricSchemas); + metricTable.setAggOption(aggOption); + List tables = new ArrayList<>(); + tables.add(metricTable); + + // 6.build ParseSqlReq + DataSetQueryParam datasetQueryParam = new DataSetQueryParam(); + datasetQueryParam.setTables(tables); + datasetQueryParam.setSql(queryStatement.getSql()); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), + database.getVersion())) { + datasetQueryParam.setSupportWith(false); + datasetQueryParam.setWithAlias(false); + } + + // 7. do deriveMetric + generateDerivedMetric(semanticSchemaResp, aggOption, datasetQueryParam); + + // 8.physicalSql by ParseSqlReq + // queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryStatement.getSql())); + queryStructReq.setDataSetId(queryStatement.getDataSetId()); + queryStructReq.setQueryType(getQueryType(aggOption)); + log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq); + QueryParam queryParam = new QueryParam(); + BeanUtils.copyProperties(queryStructReq, queryParam); + queryStatement.setQueryParam(queryParam); + queryStatement.setDataSetQueryParam(datasetQueryParam); + // queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); + } + + private AggOption getAggOption(QueryStatement queryStatement, + List metricSchemas) { + String sql = queryStatement.getSql(); + if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) + && !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) { + log.debug("getAggOption simple sql set to DEFAULT"); + return AggOption.DEFAULT; + } + // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" + // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" + if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) + || SqlSelectFunctionHelper.hasFunction(sql, "count") + || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { + return AggOption.OUTER; + } + // if (queryStatement.isInnerLayerNative()) { + // return AggOption.NATIVE; + // } + if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) + || SqlSelectHelper.hasGroupBy(sql)) { + return AggOption.OUTER; + } + long defaultAggNullCnt = metricSchemas.stream().filter( + m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg())) + .count(); + if (defaultAggNullCnt > 0) { + log.debug("getAggOption find null defaultAgg metric set to NATIVE"); + return AggOption.OUTER; + } + return AggOption.DEFAULT; + } + + private void convertNameToBizName(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); + String sql = queryStatement.getSql(); + log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(), + sql); + sql = SqlReplaceHelper.replaceSqlByPositions(sql); + log.debug("replaceSqlByPositions:{}", sql); + sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); + log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(), + sql); + sql = SqlReplaceHelper.replaceTable(sql, + Constants.TABLE_PREFIX + queryStatement.getDataSetId()); + log.debug("replaceTableName after:{}", sql); + queryStatement.setSql(sql); + } + + private Set getDimensions(SemanticSchemaResp semanticSchemaResp, + List allFields) { + Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() + .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), + SchemaItem::getBizName, (k1, k2) -> k1)); + dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(), + TimeDimensionEnum.DAY.getName()); + return allFields.stream() + .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) + .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())) + .collect(Collectors.toSet()); + } + + private List getMetrics(SemanticSchemaResp semanticSchemaResp, + List allFields) { + Map metricLowerToNameMap = + semanticSchemaResp.getMetrics().stream().collect(Collectors + .toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); + return allFields.stream() + .filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) + .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())) + .collect(Collectors.toList()); + } + + private void rewriteFunction(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + if (Objects.isNull(database) || Objects.isNull(database.getType())) { + return; + } + String type = database.getType(); + DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase()); + if (Objects.nonNull(engineAdaptor)) { + String functionNameCorrector = + engineAdaptor.functionNameCorrector(queryStatement.getSql()); + queryStatement.setSql(functionNameCorrector); + } + } + + protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { + // support fieldName and field alias to bizName + Map dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap( + entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + + Map metricResults = semanticSchemaResp.getMetrics().stream().flatMap( + entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + + dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); + dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); + dimensionResults.putAll(metricResults); + return dimensionResults; + } + + private Stream> getPairStream(String aliasStr, String name, + String bizName) { + Set> elements = new HashSet<>(); + elements.add(Pair.of(name, bizName)); + if (StringUtils.isNotBlank(aliasStr)) { + List aliasList = SchemaItem.getAliasList(aliasStr); + for (String alias : aliasList) { + elements.add(Pair.of(alias, bizName)); + } + } + return elements.stream(); + } + + private QueryType getQueryType(AggOption aggOption) { + boolean isAgg = AggOption.isAgg(aggOption); + QueryType queryType = QueryType.DETAIL; + if (isAgg) { + queryType = QueryType.AGGREGATE; + } + return queryType; + } + + private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, + DataSetQueryParam viewQueryParam) { + String sql = viewQueryParam.getSql(); + for (MetricTable metricTable : viewQueryParam.getTables()) { + Set measures = new HashSet<>(); + Map replaces = generateDerivedMetric(semanticSchemaResp, aggOption, + metricTable.getMetrics(), metricTable.getDimensions(), measures); + + if (!CollectionUtils.isEmpty(replaces)) { + // metricTable sql use measures replace metric + sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces); + metricTable.setAggOption(AggOption.NATIVE); + // metricTable use measures replace metric + if (!CollectionUtils.isEmpty(measures)) { + metricTable.setMetrics(new ArrayList<>(measures)); + } else { + // empty measure , fill default + metricTable.setMetrics(new ArrayList<>()); + metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( + getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); + } + } + } + viewQueryParam.setSql(sql); + } + + private Map generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, + AggOption aggOption, List metrics, List dimensions, + Set measures) { + Map result = new HashMap<>(); + List metricResps = semanticSchemaResp.getMetrics(); + List dimensionResps = semanticSchemaResp.getDimensions(); + + // Check if any metric is derived + boolean hasDerivedMetrics = + metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType + .isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams())); + if (!hasDerivedMetrics) { + return result; + } + + log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics); + + Set allFields = new HashSet<>(); + Map allMeasures = new HashMap<>(); + semanticSchemaResp.getModelResps().forEach(modelResp -> { + allFields.addAll(modelResp.getFieldList()); + if (modelResp.getModelDetail().getMeasures() != null) { + modelResp.getModelDetail().getMeasures() + .forEach(measure -> allMeasures.put(measure.getBizName(), measure)); + } + }); + + Set derivedDimensions = new HashSet<>(); + Set derivedMetrics = new HashSet<>(); + Map visitedMetrics = new HashMap<>(); + + for (MetricResp metricResp : metricResps) { + if (metrics.contains(metricResp.getBizName())) { + boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(), + metricResp.getMetricDefineByMeasureParams()); + if (isDerived) { + String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, + allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp), + metricResp.getMetricDefineType(), aggOption, visitedMetrics, + derivedMetrics, derivedDimensions); + result.put(metricResp.getBizName(), expr); + log.debug("derived metric {}->{}", metricResp.getBizName(), expr); + } else { + measures.add(metricResp.getBizName()); + } + } + } + + measures.addAll(derivedMetrics); + derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension)) + .forEach(dimensions::add); + + return result; + } + + private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List dimensions) { + if (!CollectionUtils.isEmpty(dimensions)) { + Map modelMatchCnt = new HashMap<>(); + for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { + modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions() + .stream().filter(d -> dimensions.contains(d.getBizName())).count()); + } + return modelMatchCnt.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) + .map(m -> m.getKey()).findFirst().orElse(""); + } + return semanticSchemaResp.getModelResps().get(0).getBizName(); + } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java index 4153f8f2e..ce0d27a11 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java @@ -1,22 +1,15 @@ package com.tencent.supersonic.headless.core.translator.calcite; -import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; -import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.QueryParser; -import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; -import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; +import com.tencent.supersonic.headless.core.translator.calcite.sql.RuntimeOptions; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.SqlBuilder; import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.parser.SqlParseException; import org.springframework.stereotype.Component; -import java.util.Collections; -import java.util.Objects; - /** the calcite parse implements */ @Component("CalciteQueryParser") @Slf4j @@ -24,55 +17,19 @@ public class CalciteQueryParser implements QueryParser { @Override public void parse(QueryStatement queryStatement, AggOption isAgg) throws Exception { - MetricQueryParam metricReq = queryStatement.getMetricQueryParam(); - SemanticModel semanticModel = queryStatement.getSemanticModel(); - if (semanticModel == null) { - queryStatement.setErrMsg("semanticSchema not found"); + Ontology ontology = queryStatement.getOntology(); + if (ontology == null) { + queryStatement.setErrMsg("No ontology could be found"); return; } - queryStatement.setMetricQueryParam(metricReq); - SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); - AggPlanner aggBuilder = new AggPlanner(semanticSchema); - aggBuilder.explain(queryStatement, isAgg); - EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType()); - queryStatement.setSql(aggBuilder.getSql(engineType)); - if (Objects.nonNull(queryStatement.getEnableOptimize()) - && queryStatement.getEnableOptimize() - && Objects.nonNull(queryStatement.getDataSetAlias()) - && !queryStatement.getDataSetAlias().isEmpty()) { - // simplify model sql with query sql - String simplifySql = aggBuilder.simplify( - getSqlByDataSet(engineType, aggBuilder.getSql(engineType), - queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()), - engineType); - if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { - log.debug("simplifySql [{}]", simplifySql); - queryStatement.setDataSetSimplifySql(simplifySql); - } - } + + S2CalciteSchema semanticSchema = S2CalciteSchema.builder() + .schemaKey("DATASET_" + queryStatement.getDataSetId()).ontology(ontology) + .runtimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) + .enableOptimize(queryStatement.getEnableOptimize()).build()) + .build(); + SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema); + sqlBuilder.build(queryStatement, isAgg); } - private SemanticSchema getSemanticSchema(SemanticModel semanticModel, - QueryStatement queryStatement) { - SemanticSchema semanticSchema = - SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build(); - semanticSchema.setSemanticModel(semanticModel); - semanticSchema.setDatasource(semanticModel.getDatasourceMap()); - semanticSchema.setDimension(semanticModel.getDimensionMap()); - semanticSchema.setMetric(semanticModel.getMetrics()); - semanticSchema.setJoinRelations(semanticModel.getJoinRelations()); - semanticSchema.setRuntimeOptions( - RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) - .enableOptimize(queryStatement.getEnableOptimize()).build()); - return semanticSchema; - } - - private String getSqlByDataSet(EngineType engineType, String parentSql, String dataSetSql, - String parentAlias) throws SqlParseException { - if (!SqlMergeWithUtils.hasWith(engineType, dataSetSql)) { - return String.format("with %s as (%s) %s", parentAlias, parentSql, dataSetSql); - } - return SqlMergeWithUtils.mergeWith(engineType, dataSetSql, - Collections.singletonList(parentSql), Collections.singletonList(parentAlias)); - } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java deleted file mode 100644 index b39e6e5af..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.calcite.planner; - -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; - -/** parse and generate SQL and other execute information */ -public interface Planner { - - public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception; - - public String getSql(EngineType enginType); - - public String getSourceId(); - - public String simplify(String sql, EngineType engineType); -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataModel.java similarity index 90% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataModel.java index 300744284..1de81b929 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataModel.java @@ -7,13 +7,13 @@ import java.util.List; @Data @Builder -public class DataSource { +public class DataModel { private Long id; private String name; - private Long sourceId; + private Long modelId; private String type; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java index 98aa5b8f3..559444f32 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Dimension.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticItem; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java index 366f2c60c..d58328ec3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Metric.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticItem; import lombok.Data; import java.util.List; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Ontology.java similarity index 71% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Ontology.java index 40b4e8171..da2e21698 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Ontology.java @@ -11,11 +11,10 @@ import java.util.Map; import java.util.stream.Collectors; @Data -public class SemanticModel { +public class Ontology { - private String schemaKey; private List metrics = new ArrayList<>(); - private Map datasourceMap = new HashMap<>(); + private Map dataModelMap = new HashMap<>(); private Map> dimensionMap = new HashMap<>(); private List materializationList = new ArrayList<>(); private List joinRelations; @@ -26,8 +25,8 @@ public class SemanticModel { .collect(Collectors.toList()); } - public Map getModelMap() { - return datasourceMap.values().stream() - .collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource)); + public Map getModelMap() { + return dataModelMap.values().stream() + .collect(Collectors.toMap(DataModel::getId, dataSource -> dataSource)); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticItem.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticItem.java similarity index 66% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticItem.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticItem.java index 8d03edf65..be239b1d7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticItem.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticItem.java @@ -1,6 +1,7 @@ -package com.tencent.supersonic.headless.core.translator.calcite.schema; +package com.tencent.supersonic.headless.core.translator.calcite.s2sql; public interface SemanticItem { + String getName(); - public String getName(); + String getType(); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java deleted file mode 100644 index c8afc0cd7..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java +++ /dev/null @@ -1,136 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.calcite.schema; - -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.SchemaVersion; -import org.apache.calcite.schema.Table; -import org.apache.calcite.schema.impl.AbstractSchema; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class SemanticSchema extends AbstractSchema { - - private final String schemaKey; - private final Map tableMap; - - private SemanticModel semanticModel = new SemanticModel(); - - private List joinRelations; - - private RuntimeOptions runtimeOptions; - - private SemanticSchema(String schemaKey, Map tableMap) { - this.schemaKey = schemaKey; - this.tableMap = tableMap; - } - - public static Builder newBuilder(String schemaKey) { - return new Builder(schemaKey); - } - - public String getSchemaKey() { - return schemaKey; - } - - public void setSemanticModel(SemanticModel semanticModel) { - this.semanticModel = semanticModel; - } - - public SemanticModel getSemanticModel() { - return semanticModel; - } - - @Override - public Map getTableMap() { - return tableMap; - } - - @Override - public Schema snapshot(SchemaVersion version) { - return this; - } - - public Map getDatasource() { - return semanticModel.getDatasourceMap(); - } - - public void setDatasource(Map datasource) { - semanticModel.setDatasourceMap(datasource); - } - - public Map> getDimension() { - return semanticModel.getDimensionMap(); - } - - public void setDimension(Map> dimensions) { - semanticModel.setDimensionMap(dimensions); - } - - public List getMetrics() { - return semanticModel.getMetrics(); - } - - public void setMetric(List metric) { - semanticModel.setMetrics(metric); - } - - public void setMaterializationList(List materializationList) { - semanticModel.setMaterializationList(materializationList); - } - - public List getMaterializationList() { - return semanticModel.getMaterializationList(); - } - - public void setJoinRelations(List joinRelations) { - this.joinRelations = joinRelations; - } - - public List getJoinRelations() { - return joinRelations; - } - - public void setRuntimeOptions(RuntimeOptions runtimeOptions) { - this.runtimeOptions = runtimeOptions; - } - - public RuntimeOptions getRuntimeOptions() { - return runtimeOptions; - } - - public static final class Builder { - - private final String schemaKey; - private final Map tableMap = new HashMap<>(); - - private Builder(String schemaKey) { - if (schemaKey == null) { - throw new IllegalArgumentException("Schema name cannot be null or empty"); - } - - this.schemaKey = schemaKey; - } - - public Builder addTable(DataSourceTable table) { - if (tableMap.containsKey(table.getTableName())) { - throw new IllegalArgumentException( - "Table already defined: " + table.getTableName()); - } - - tableMap.put(table.getTableName(), table); - - return this; - } - - public SemanticSchema build() { - return new SemanticSchema(schemaKey, tableMap); - } - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/FilterToGroupScanRule.java similarity index 87% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/FilterToGroupScanRule.java index b812f8f8c..59e163791 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/FilterToGroupScanRule.java @@ -1,6 +1,5 @@ -package com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer; +package com.tencent.supersonic.headless.core.translator.calcite.sql; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; import org.apache.calcite.rel.core.Aggregate; @@ -40,23 +39,23 @@ public class FilterToGroupScanRule extends RelRule implements Transforma }); }).as(FilterTableScanRule.Config.class); - private SemanticSchema semanticSchema; + private S2CalciteSchema schema; - public FilterToGroupScanRule(FilterTableScanRule.Config config, SemanticSchema semanticSchema) { + public FilterToGroupScanRule(FilterTableScanRule.Config config, S2CalciteSchema schema) { super(config); - this.semanticSchema = semanticSchema; + this.schema = schema; } public void onMatch(RelOptRuleCall call) { if (call.rels.length != 4) { return; } - if (Objects.isNull(semanticSchema.getRuntimeOptions()) - || Objects.isNull(semanticSchema.getRuntimeOptions().getMinMaxTime()) - || semanticSchema.getRuntimeOptions().getMinMaxTime().getLeft().isEmpty()) { + if (Objects.isNull(schema.getRuntimeOptions()) + || Objects.isNull(schema.getRuntimeOptions().getMinMaxTime()) + || schema.getRuntimeOptions().getMinMaxTime().getLeft().isEmpty()) { return; } - Triple minMax = semanticSchema.getRuntimeOptions().getMinMaxTime(); + Triple minMax = schema.getRuntimeOptions().getMinMaxTime(); Filter filter = (Filter) call.rel(0); Project project0 = (Project) call.rel(1); Project project1 = (Project) call.rel(3); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java deleted file mode 100644 index 6df9889c2..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.calcite.sql; - -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; - -public interface Optimization { - - public void visit(SemanticSchema semanticSchema); -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/RuntimeOptions.java similarity index 97% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/RuntimeOptions.java index 1913f17a3..9bf681b3e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/RuntimeOptions.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/RuntimeOptions.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.calcite.schema; +package com.tencent.supersonic.headless.core.translator.calcite.sql; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteSchema.java new file mode 100644 index 000000000..e1559809f --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteSchema.java @@ -0,0 +1,48 @@ +package com.tencent.supersonic.headless.core.translator.calcite.sql; + +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; +import lombok.Builder; +import lombok.Data; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaVersion; +import org.apache.calcite.schema.impl.AbstractSchema; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +public class S2CalciteSchema extends AbstractSchema { + + private String schemaKey; + + private Ontology ontology; + + private RuntimeOptions runtimeOptions; + + @Override + public Schema snapshot(SchemaVersion version) { + return this; + } + + public Map getDataModels() { + return ontology.getDataModelMap(); + } + + public List getMetrics() { + return ontology.getMetrics(); + } + + public Map> getDimensions() { + return ontology.getDimensionMap(); + } + + public List getJoinRelations() { + return ontology.getJoinRelations(); + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteTable.java similarity index 92% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteTable.java index e5e11f6dc..98a2b08fc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/DataSourceTable.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteTable.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.calcite.schema; +package com.tencent.supersonic.headless.core.translator.calcite.sql; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.Enumerable; @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.List; /** customize the AbstractTable */ -public class DataSourceTable extends AbstractTable implements ScannableTable, TranslatableTable { +public class S2CalciteTable extends AbstractTable implements ScannableTable, TranslatableTable { private final String tableName; private final List fieldNames; @@ -32,7 +32,7 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr private RelDataType rowType; - private DataSourceTable(String tableName, List fieldNames, List fieldTypes, + private S2CalciteTable(String tableName, List fieldNames, List fieldTypes, Statistic statistic) { this.tableName = tableName; this.fieldNames = fieldNames; @@ -116,7 +116,7 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr return this; } - public DataSourceTable build() { + public S2CalciteTable build() { if (fieldNames.isEmpty()) { throw new IllegalStateException("Table must have at least one field"); } @@ -125,7 +125,7 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr throw new IllegalStateException("Table must have positive row count"); } - return new DataSourceTable(tableName, fieldNames, fieldTypes, + return new S2CalciteTable(tableName, fieldNames, fieldTypes, Statistics.of(rowCount, null)); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SchemaBuilder.java similarity index 85% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SchemaBuilder.java index 8cad09dac..43949e000 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SchemaBuilder.java @@ -1,8 +1,7 @@ -package com.tencent.supersonic.headless.core.translator.calcite.schema; +package com.tencent.supersonic.headless.core.translator.calcite.sql; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.translator.calcite.sql.S2SQLSqlValidatorImpl; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.prepare.CalciteCatalogReader; import org.apache.calcite.prepare.Prepare; @@ -27,15 +26,14 @@ public class SchemaBuilder { public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1"; public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2"; - public static SqlValidatorScope getScope(SemanticSchema schema) throws Exception { + public static SqlValidatorScope getScope(S2CalciteSchema schema) throws Exception { Map nameToTypeMap = new HashMap<>(); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); rootSchema.add(schema.getSchemaKey(), schema); Prepare.CatalogReader catalogReader = new CalciteCatalogReader(rootSchema, Collections.singletonList(schema.getSchemaKey()), Configuration.typeFactory, Configuration.config); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); S2SQLSqlValidatorImpl s2SQLSqlValidator = new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader, Configuration.typeFactory, Configuration.getValidatorConfig(engineType)); @@ -45,12 +43,12 @@ public class SchemaBuilder { public static CalciteSchema getMaterializationSchema() { CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); SchemaPlus schema = rootSchema.plus().add(MATERIALIZATION_SYS_DB, new AbstractSchema()); - DataSourceTable srcTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_SOURCE) + S2CalciteTable srcTable = S2CalciteTable.newBuilder(MATERIALIZATION_SYS_SOURCE) .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT).withRowCount(1) .build(); schema.add(MATERIALIZATION_SYS_SOURCE, srcTable); - DataSourceTable dataSetTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW) + S2CalciteTable dataSetTable = S2CalciteTable.newBuilder(MATERIALIZATION_SYS_VIEW) .addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE) .addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT).withRowCount(1) .build(); @@ -62,7 +60,7 @@ public class SchemaBuilder { Set dates, Set dimensions, Set metrics) { String tb = tbSrc; String db = dbSrc; - DataSourceTable.Builder builder = DataSourceTable.newBuilder(tb); + S2CalciteTable.Builder builder = S2CalciteTable.newBuilder(tb); for (String date : dates) { builder.addField(date, SqlTypeName.VARCHAR); } @@ -72,7 +70,7 @@ public class SchemaBuilder { for (String metric : metrics) { builder.addField(metric, SqlTypeName.ANY); } - DataSourceTable srcTable = builder.withRowCount(1).build(); + S2CalciteTable srcTable = builder.withRowCount(1).build(); if (Objects.nonNull(db) && !db.isEmpty()) { SchemaPlus dbPs = dataSetSchema.plus(); for (String d : db.split("\\.")) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java similarity index 51% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java index fd12cf821..971886ed4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java @@ -1,60 +1,91 @@ -package com.tencent.supersonic.headless.core.translator.calcite.planner; +package com.tencent.supersonic.headless.core.translator.calcite.sql; import com.tencent.supersonic.common.calcite.Configuration; +import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; -import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; -import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; +import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.OutputRender; +import com.tencent.supersonic.headless.core.translator.calcite.sql.render.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.SourceRender; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.validate.SqlValidatorScope; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import java.util.Objects; -import java.util.Stack; /** parsing from query dimensions and metrics */ @Slf4j -public class AggPlanner implements Planner { +public class SqlBuilder { - private MetricQueryParam metricReq; - private SemanticSchema schema; + private final S2CalciteSchema schema; + private MetricQueryParam metricQueryParam; private SqlValidatorScope scope; - private Stack dataSets = new Stack<>(); private SqlNode parserNode; - private String sourceId; private boolean isAgg = false; private AggOption aggOption = AggOption.DEFAULT; - public AggPlanner(SemanticSchema schema) { + public SqlBuilder(S2CalciteSchema schema) { this.schema = schema; } - public void parse() throws Exception { + public void build(QueryStatement queryStatement, AggOption aggOption) throws Exception { + this.metricQueryParam = queryStatement.getMetricQueryParam(); + if (metricQueryParam.getMetrics() == null) { + metricQueryParam.setMetrics(new ArrayList<>()); + } + if (metricQueryParam.getDimensions() == null) { + metricQueryParam.setDimensions(new ArrayList<>()); + } + if (metricQueryParam.getLimit() == null) { + metricQueryParam.setLimit(0L); + } + this.aggOption = aggOption; + + buildParseNode(); + Database database = queryStatement.getOntology().getDatabase(); + EngineType engineType = EngineType.fromString(database.getType()); + optimizeParseNode(engineType); + String sql = getSql(engineType); + + queryStatement.setSql(sql); + if (Objects.nonNull(queryStatement.getEnableOptimize()) + && queryStatement.getEnableOptimize() + && Objects.nonNull(queryStatement.getDataSetAlias()) + && !queryStatement.getDataSetAlias().isEmpty()) { + // simplify model sql with query sql + String simplifySql = rewrite(getSqlByDataSet(engineType, sql, + queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()), engineType); + if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { + log.debug("simplifySql [{}]", simplifySql); + queryStatement.setDataSetSimplifySql(simplifySql); + } + } + } + + private void buildParseNode() throws Exception { // find the match Datasource scope = SchemaBuilder.getScope(schema); - List datasource = getMatchDataSource(scope); - if (datasource == null || datasource.isEmpty()) { - throw new Exception("datasource not found"); + List dataModels = + DataModelNode.getRelatedDataModels(scope, schema, metricQueryParam); + if (dataModels == null || dataModels.isEmpty()) { + throw new Exception("data model not found"); } - isAgg = getAgg(datasource.get(0)); - sourceId = String.valueOf(datasource.get(0).getSourceId()); + isAgg = getAgg(dataModels.get(0)); // build level by level LinkedList builders = new LinkedList<>(); @@ -67,84 +98,36 @@ public class AggPlanner implements Planner { while (it.hasNext()) { Renderer renderer = it.next(); if (previous != null) { - previous.render(metricReq, datasource, scope, schema, !isAgg); + previous.render(metricQueryParam, dataModels, scope, schema, !isAgg); renderer.setTable(previous - .builderAs(DataSourceNode.getNames(datasource) + "_" + String.valueOf(i))); + .builderAs(DataModelNode.getNames(dataModels) + "_" + String.valueOf(i))); i++; } previous = renderer; } - builders.getLast().render(metricReq, datasource, scope, schema, !isAgg); + builders.getLast().render(metricQueryParam, dataModels, scope, schema, !isAgg); parserNode = builders.getLast().builder(); } - private List getMatchDataSource(SqlValidatorScope scope) throws Exception { - return DataSourceNode.getMatchDataSources(scope, schema, metricReq); - } - - private boolean getAgg(DataSource dataSource) { + private boolean getAgg(DataModel dataModel) { if (!AggOption.DEFAULT.equals(aggOption)) { return AggOption.isAgg(aggOption); } - // default by dataSource time aggregation - if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime() + // default by dataModel time aggregation + if (Objects.nonNull(dataModel.getAggTime()) && !dataModel.getAggTime() .equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { - if (!metricReq.isNativeQuery()) { + if (!metricQueryParam.isNativeQuery()) { return true; } } return isAgg; } - @Override - public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception { - this.metricReq = queryStatement.getMetricQueryParam(); - if (metricReq.getMetrics() == null) { - metricReq.setMetrics(new ArrayList<>()); - } - if (metricReq.getDimensions() == null) { - metricReq.setDimensions(new ArrayList<>()); - } - if (metricReq.getLimit() == null) { - metricReq.setLimit(0L); - } - this.aggOption = aggOption; - // build a parse Node - parse(); - // optimizer - Database database = queryStatement.getSemanticModel().getDatabase(); - EngineType engineType = EngineType.fromString(database.getType()); - optimize(engineType); - } - - @Override public String getSql(EngineType engineType) { return SemanticNode.getSql(parserNode, engineType); } - @Override - public String getSourceId() { - return sourceId; - } - - @Override - public String simplify(String sql, EngineType engineType) { - return optimize(sql, engineType); - } - - public void optimize(EngineType engineType) { - if (Objects.isNull(schema.getRuntimeOptions()) - || Objects.isNull(schema.getRuntimeOptions().getEnableOptimize()) - || !schema.getRuntimeOptions().getEnableOptimize()) { - return; - } - SqlNode optimizeNode = optimizeSql(SemanticNode.getSql(parserNode, engineType), engineType); - if (Objects.nonNull(optimizeNode)) { - parserNode = optimizeNode; - } - } - - public String optimize(String sql, EngineType engineType) { + private String rewrite(String sql, EngineType engineType) { try { SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); @@ -153,21 +136,41 @@ public class AggPlanner implements Planner { SemanticNode.optimize(scope, schema, sqlNode, engineType), engineType); } } catch (Exception e) { - log.error("optimize error {}", e); + log.error("optimize error {}", e.toString()); } return ""; } - private SqlNode optimizeSql(String sql, EngineType engineType) { + private void optimizeParseNode(EngineType engineType) { + if (Objects.isNull(schema.getRuntimeOptions()) + || Objects.isNull(schema.getRuntimeOptions().getEnableOptimize()) + || !schema.getRuntimeOptions().getEnableOptimize()) { + return; + } + + SqlNode optimizeNode = null; try { - SqlNode sqlNode = - SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); + SqlNode sqlNode = SqlParser.create(SemanticNode.getSql(parserNode, engineType), + Configuration.getParserConfig(engineType)).parseStmt(); if (Objects.nonNull(sqlNode)) { - return SemanticNode.optimize(scope, schema, sqlNode, engineType); + optimizeNode = SemanticNode.optimize(scope, schema, sqlNode, engineType); } } catch (Exception e) { log.error("optimize error {}", e); } - return null; + + if (Objects.nonNull(optimizeNode)) { + parserNode = optimizeNode; + } } + + private String getSqlByDataSet(EngineType engineType, String parentSql, String dataSetSql, + String parentAlias) throws SqlParseException { + if (!SqlMergeWithUtils.hasWith(engineType, dataSetSql)) { + return String.format("with %s as (%s) %s", parentAlias, parentSql, dataSetSql); + } + return SqlMergeWithUtils.mergeWith(engineType, dataSetSql, + Collections.singletonList(parentSql), Collections.singletonList(parentAlias)); + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java index 8c21132f9..74828fbc2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import lombok.Data; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlKind; @@ -27,7 +27,7 @@ public class TableView { private String alias; private List primary; - private DataSource dataSource; + private DataModel dataModel; public SqlNode build() { measure.addAll(dimension); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java similarity index 77% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java index ae3fab3a5..73c5c422e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java @@ -6,14 +6,13 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.node.extend.LateralViewExplodeNode; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.SchemaBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDataTypeSpec; @@ -38,30 +37,30 @@ import java.util.Set; import java.util.stream.Collectors; @Slf4j -public class DataSourceNode extends SemanticNode { +public class DataModelNode extends SemanticNode { - public static SqlNode build(DataSource datasource, SqlValidatorScope scope) throws Exception { + public static SqlNode build(DataModel dataModel, SqlValidatorScope scope) throws Exception { String sqlTable = ""; - if (datasource.getSqlQuery() != null && !datasource.getSqlQuery().isEmpty()) { - sqlTable = datasource.getSqlQuery(); - } else if (datasource.getTableQuery() != null && !datasource.getTableQuery().isEmpty()) { - if (datasource.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { - String fullTableName = Arrays.stream(datasource.getTableQuery().split("\\.")) + if (dataModel.getSqlQuery() != null && !dataModel.getSqlQuery().isEmpty()) { + sqlTable = dataModel.getSqlQuery(); + } else if (dataModel.getTableQuery() != null && !dataModel.getTableQuery().isEmpty()) { + if (dataModel.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { + String fullTableName = Arrays.stream(dataModel.getTableQuery().split("\\.")) .collect(Collectors.joining(".public.")); sqlTable = "select * from " + fullTableName; } else { - sqlTable = "select * from " + datasource.getTableQuery(); + sqlTable = "select * from " + dataModel.getTableQuery(); } } if (sqlTable.isEmpty()) { throw new Exception("DatasourceNode build error [tableSqlNode not found]"); } - SqlNode source = getTable(sqlTable, scope, EngineType.fromString(datasource.getType())); - addSchema(scope, datasource, sqlTable); - return buildAs(datasource.getName(), source); + SqlNode source = getTable(sqlTable, scope, EngineType.fromString(dataModel.getType())); + addSchema(scope, dataModel, sqlTable); + return buildAs(dataModel.getName(), source); } - private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) + private static void addSchema(SqlValidatorScope scope, DataModel datasource, String table) throws Exception { Map> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table); for (Map.Entry> entry : sqlTable.entrySet()) { @@ -75,7 +74,7 @@ public class DataSourceNode extends SemanticNode { } } - private static void addSchemaTable(SqlValidatorScope scope, DataSource datasource, String db, + private static void addSchemaTable(SqlValidatorScope scope, DataModel datasource, String db, String tb, Set fields) throws Exception { Set dateInfo = new HashSet<>(); Set dimensions = new HashSet<>(); @@ -112,7 +111,7 @@ public class DataSourceNode extends SemanticNode { dateInfo, dimensions, metrics); } - public static SqlNode buildExtend(DataSource datasource, Map exprList, + public static SqlNode buildExtend(DataModel datasource, Map exprList, SqlValidatorScope scope) throws Exception { if (CollectionUtils.isEmpty(exprList)) { return build(datasource, scope); @@ -146,11 +145,11 @@ public class DataSourceNode extends SemanticNode { return sqlNode; } - public static String getNames(List dataSourceList) { - return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); + public static String getNames(List dataModelList) { + return dataModelList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); } - public static void getQueryDimensionMeasure(SemanticSchema schema, + public static void getQueryDimensionMeasure(S2CalciteSchema schema, MetricQueryParam metricCommand, Set queryDimension, List measures) { queryDimension.addAll(metricCommand.getDimensions().stream() .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) @@ -166,11 +165,10 @@ public class DataSourceNode extends SemanticNode { .forEach(m -> measures.add(m)); } - public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, + public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema, MetricQueryParam metricCommand, Set queryDimension, List measures, SqlValidatorScope scope) throws Exception { - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { Set filterConditions = new HashSet<>(); FilterNode.getFilterField(parse(metricCommand.getWhere(), scope, engineType), @@ -193,18 +191,18 @@ public class DataSourceNode extends SemanticNode { } } - public static List getMatchDataSources(SqlValidatorScope scope, - SemanticSchema schema, MetricQueryParam metricCommand) throws Exception { - List dataSources = new ArrayList<>(); + public static List getRelatedDataModels(SqlValidatorScope scope, + S2CalciteSchema schema, MetricQueryParam metricCommand) throws Exception { + List dataModels = new ArrayList<>(); // check by metric List measures = new ArrayList<>(); Set queryDimension = new HashSet<>(); getQueryDimensionMeasure(schema, metricCommand, queryDimension, measures); - DataSource baseDataSource = null; + DataModel baseDataModel = null; // one , match measure count Map dataSourceMeasures = new HashMap<>(); - for (Map.Entry entry : schema.getDatasource().entrySet()) { + for (Map.Entry entry : schema.getDataModels().entrySet()) { Set sourceMeasure = entry.getValue().getMeasures().stream() .map(mm -> mm.getName()).collect(Collectors.toSet()); sourceMeasure.retainAll(measures); @@ -214,58 +212,58 @@ public class DataSourceNode extends SemanticNode { Optional> base = dataSourceMeasures.entrySet().stream() .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst(); if (base.isPresent()) { - baseDataSource = schema.getDatasource().get(base.get().getKey()); - dataSources.add(baseDataSource); + baseDataModel = schema.getDataModels().get(base.get().getKey()); + dataModels.add(baseDataModel); } // second , check match all dimension and metric - if (baseDataSource != null) { + if (baseDataModel != null) { Set filterMeasure = new HashSet<>(); - Set sourceMeasure = baseDataSource.getMeasures().stream() - .map(mm -> mm.getName()).collect(Collectors.toSet()); - Set dimension = baseDataSource.getDimensions().stream().map(dd -> dd.getName()) + Set sourceMeasure = baseDataModel.getMeasures().stream().map(mm -> mm.getName()) .collect(Collectors.toSet()); - baseDataSource.getIdentifiers().stream().forEach(i -> dimension.add(i.getName())); - if (schema.getDimension().containsKey(baseDataSource.getName())) { - schema.getDimension().get(baseDataSource.getName()).stream() + Set dimension = baseDataModel.getDimensions().stream().map(dd -> dd.getName()) + .collect(Collectors.toSet()); + baseDataModel.getIdentifiers().stream().forEach(i -> dimension.add(i.getName())); + if (schema.getDimensions().containsKey(baseDataModel.getName())) { + schema.getDimensions().get(baseDataModel.getName()).stream() .forEach(d -> dimension.add(d.getName())); } filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(dimension); EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType.fromString(schema.getOntology().getDatabase().getType()); mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures, scope); boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, metricCommand, scope, engineType); if (isAllMatch) { - log.debug("baseDataSource match all "); - return dataSources; + log.debug("baseDataModel match all "); + return dataModels; } // find all dataSource has the same identifiers - List linkDataSources = getLinkDataSourcesByJoinRelation(queryDimension, - measures, baseDataSource, schema); - if (CollectionUtils.isEmpty(linkDataSources)) { - log.debug("baseDataSource get by identifiers "); - Set baseIdentifiers = baseDataSource.getIdentifiers().stream() + List linkDataModels = getLinkDataSourcesByJoinRelation(queryDimension, + measures, baseDataModel, schema); + if (CollectionUtils.isEmpty(linkDataModels)) { + log.debug("baseDataModel get by identifiers "); + Set baseIdentifiers = baseDataModel.getIdentifiers().stream() .map(i -> i.getName()).collect(Collectors.toSet()); if (baseIdentifiers.isEmpty()) { throw new Exception( - "datasource error : " + baseDataSource.getName() + " miss identifier"); + "datasource error : " + baseDataModel.getName() + " miss identifier"); } - linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures, - baseDataSource, schema); - if (linkDataSources.isEmpty()) { + linkDataModels = getLinkDataSources(baseIdentifiers, queryDimension, measures, + baseDataModel, schema); + if (linkDataModels.isEmpty()) { throw new Exception(String.format( "not find the match datasource : dimension[%s],measure[%s]", queryDimension, measures)); } } - log.debug("linkDataSources {}", linkDataSources); - return linkDataSources; - // dataSources.addAll(linkDataSources); + log.debug("linkDataModels {}", linkDataModels); + return linkDataModels; + // dataModels.addAll(linkDataModels); } - return dataSources; + return dataModels; } private static boolean checkMatch(Set sourceMeasure, Set queryDimension, @@ -301,17 +299,17 @@ public class DataSourceNode extends SemanticNode { return isAllMatch; } - private static List getLinkDataSourcesByJoinRelation(Set queryDimension, - List measures, DataSource baseDataSource, SemanticSchema schema) { + private static List getLinkDataSourcesByJoinRelation(Set queryDimension, + List measures, DataModel baseDataModel, S2CalciteSchema schema) { Set linkDataSourceName = new HashSet<>(); - List linkDataSources = new ArrayList<>(); + List linkDataModels = new ArrayList<>(); Set before = new HashSet<>(); - before.add(baseDataSource.getName()); + before.add(baseDataModel.getName()); if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { Set visitJoinRelations = new HashSet<>(); List sortedJoinRelation = new ArrayList<>(); - sortJoinRelation(schema.getJoinRelations(), baseDataSource.getName(), - visitJoinRelations, sortedJoinRelation); + sortJoinRelation(schema.getJoinRelations(), baseDataModel.getName(), visitJoinRelations, + sortedJoinRelation); schema.getJoinRelations().stream().filter(j -> !visitJoinRelations.contains(j.getId())) .forEach(j -> sortedJoinRelation.add(j)); for (JoinRelation joinRelation : sortedJoinRelation) { @@ -321,8 +319,8 @@ public class DataSourceNode extends SemanticNode { } boolean isMatch = false; boolean isRight = before.contains(joinRelation.getLeft()); - DataSource other = isRight ? schema.getDatasource().get(joinRelation.getRight()) - : schema.getDatasource().get(joinRelation.getLeft()); + DataModel other = isRight ? schema.getDataModels().get(joinRelation.getRight()) + : schema.getDataModels().get(joinRelation.getLeft()); if (!queryDimension.isEmpty()) { Set linkDimension = other.getDimensions().stream() .map(dd -> dd.getName()).collect(Collectors.toSet()); @@ -338,8 +336,8 @@ public class DataSourceNode extends SemanticNode { if (!linkMeasure.isEmpty()) { isMatch = true; } - if (!isMatch && schema.getDimension().containsKey(other.getName())) { - Set linkDimension = schema.getDimension().get(other.getName()).stream() + if (!isMatch && schema.getDimensions().containsKey(other.getName())) { + Set linkDimension = schema.getDimensions().get(other.getName()).stream() .map(dd -> dd.getName()).collect(Collectors.toSet()); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { @@ -354,8 +352,8 @@ public class DataSourceNode extends SemanticNode { } if (!CollectionUtils.isEmpty(linkDataSourceName)) { Map orders = new HashMap<>(); - linkDataSourceName.add(baseDataSource.getName()); - orders.put(baseDataSource.getName(), 0L); + linkDataSourceName.add(baseDataModel.getName()); + orders.put(baseDataModel.getName(), 0L); for (JoinRelation joinRelation : schema.getJoinRelations()) { if (linkDataSourceName.contains(joinRelation.getLeft()) && linkDataSourceName.contains(joinRelation.getRight())) { @@ -364,10 +362,10 @@ public class DataSourceNode extends SemanticNode { } } orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> { - linkDataSources.add(schema.getDatasource().get(d.getKey())); + linkDataModels.add(schema.getDataModels().get(d.getKey())); }); } - return linkDataSources; + return linkDataModels; } private static void sortJoinRelation(List joinRelations, String next, @@ -385,13 +383,13 @@ public class DataSourceNode extends SemanticNode { } } - private static List getLinkDataSources(Set baseIdentifiers, - Set queryDimension, List measures, DataSource baseDataSource, - SemanticSchema schema) { + private static List getLinkDataSources(Set baseIdentifiers, + Set queryDimension, List measures, DataModel baseDataModel, + S2CalciteSchema schema) { Set linkDataSourceName = new HashSet<>(); - List linkDataSources = new ArrayList<>(); - for (Map.Entry entry : schema.getDatasource().entrySet()) { - if (entry.getKey().equalsIgnoreCase(baseDataSource.getName())) { + List linkDataModels = new ArrayList<>(); + for (Map.Entry entry : schema.getDataModels().entrySet()) { + if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) { continue; } Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName()) @@ -421,7 +419,7 @@ public class DataSourceNode extends SemanticNode { } } } - for (Map.Entry> entry : schema.getDimension().entrySet()) { + for (Map.Entry> entry : schema.getDimensions().entrySet()) { if (!queryDimension.isEmpty()) { Set linkDimension = entry.getValue().stream().map(dd -> dd.getName()) .collect(Collectors.toSet()); @@ -432,12 +430,12 @@ public class DataSourceNode extends SemanticNode { } } for (String linkName : linkDataSourceName) { - linkDataSources.add(schema.getDatasource().get(linkName)); + linkDataModels.add(schema.getDataModels().get(linkName)); } - if (!CollectionUtils.isEmpty(linkDataSources)) { - List all = new ArrayList<>(); - all.add(baseDataSource); - all.addAll(linkDataSources); + if (!CollectionUtils.isEmpty(linkDataModels)) { + List all = new ArrayList<>(); + all.add(baseDataModel); + all.addAll(linkDataModels); return all; } return Lists.newArrayList(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/JoinNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/JoinNode.java deleted file mode 100644 index 2819942a8..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/JoinNode.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.calcite.sql.node; - -import lombok.Data; -import org.apache.calcite.sql.SqlNode; - -@Data -public class JoinNode extends SemanticNode { - - private SqlNode join; - private SqlNode on; - private SqlNode left; - private SqlNode right; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/LateralViewExplodeNode.java similarity index 96% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/LateralViewExplodeNode.java index d51847e7a..920bd9a1c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/extend/LateralViewExplodeNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/LateralViewExplodeNode.java @@ -1,6 +1,5 @@ -package com.tencent.supersonic.headless.core.translator.calcite.sql.node.extend; +package com.tencent.supersonic.headless.core.translator.calcite.sql.node; -import com.tencent.supersonic.headless.core.translator.calcite.sql.node.ExtendNode; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java index 6a894452b..fe4ac64d5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import lombok.Data; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; @@ -30,7 +30,7 @@ public class MetricNode extends SemanticNode { return buildAs(metric.getName(), sqlNode); } - public static Boolean isMetricField(String name, SemanticSchema schema) { + public static Boolean isMetricField(String name, S2CalciteSchema schema) { Optional metric = schema.getMetrics().stream() .filter(m -> m.getName().equalsIgnoreCase(name)).findFirst(); return metric.isPresent() && metric.get().getMetricTypeParams().isFieldMetric(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java index 78cc8720e..4f3849e2b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java @@ -5,8 +5,8 @@ import com.tencent.supersonic.common.calcite.SemanticSqlDialect; import com.tencent.supersonic.common.calcite.SqlDialectFactory; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer.FilterToGroupScanRule; +import com.tencent.supersonic.headless.core.translator.calcite.sql.FilterToGroupScanRule; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.hep.HepPlanner; @@ -397,7 +397,7 @@ public abstract class SemanticNode { return parseInfo; } - public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode, + public static SqlNode optimize(SqlValidatorScope scope, S2CalciteSchema schema, SqlNode sqlNode, EngineType engineType) { try { HepProgramBuilder hepProgramBuilder = new HepProgramBuilder(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java index bf3abfd03..fd058739a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java @@ -3,10 +3,9 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; @@ -27,14 +26,13 @@ import java.util.stream.Collectors; public class FilterRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataSources, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { + public void render(MetricQueryParam metricCommand, List dataModels, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView tableView = super.tableView; SqlNode filterNode = null; List queryMetrics = new ArrayList<>(metricCommand.getMetrics()); List queryDimensions = new ArrayList<>(metricCommand.getDimensions()); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); if (metricCommand.getWhere() != null && !metricCommand.getWhere().isEmpty()) { filterNode = SemanticNode.parse(metricCommand.getWhere(), scope, engineType); @@ -43,9 +41,9 @@ public class FilterRender extends Renderer { List fieldWhere = whereFields.stream().collect(Collectors.toList()); Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); - for (DataSource dataSource : dataSources) { + for (DataModel dataModel : dataModels) { SourceRender.whereDimMetric(fieldWhere, metricCommand.getMetrics(), - metricCommand.getDimensions(), dataSource, schema, dimensions, metrics); + metricCommand.getDimensions(), dataModel, schema, dimensions, metrics); } queryMetrics.addAll(metrics); queryDimensions.addAll(dimensions); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java index 76cc6bf68..a74b85934 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java @@ -3,17 +3,16 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode; -import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; +import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.IdentifyNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; @@ -48,11 +47,10 @@ import java.util.stream.Collectors; public class JoinRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataSources, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { + public void render(MetricQueryParam metricCommand, List dataModels, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); if (queryWhere != null && !queryWhere.isEmpty()) { @@ -62,7 +60,7 @@ public class JoinRender extends Renderer { } Set queryAllDimension = new HashSet<>(); List measures = new ArrayList<>(); - DataSourceNode.getQueryDimensionMeasure(schema, metricCommand, queryAllDimension, measures); + DataModelNode.getQueryDimensionMeasure(schema, metricCommand, queryAllDimension, measures); SqlNode left = null; TableView leftTable = null; TableView innerView = new TableView(); @@ -71,14 +69,14 @@ public class JoinRender extends Renderer { Set filterDimension = new HashSet<>(); Map beforeSources = new HashMap<>(); - for (int i = 0; i < dataSources.size(); i++) { - final DataSource dataSource = dataSources.get(i); + for (int i = 0; i < dataModels.size(); i++) { + final DataModel dataModel = dataModels.get(i); final Set filterDimensions = new HashSet<>(); final Set filterMetrics = new HashSet<>(); final List queryDimension = new ArrayList<>(); final List queryMetrics = new ArrayList<>(); - SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataSource, - schema, filterDimensions, filterMetrics); + SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataModel, schema, + filterDimensions, filterMetrics); List reqMetric = new ArrayList<>(metricCommand.getMetrics()); reqMetric.addAll(filterMetrics); reqMetric = uniqList(reqMetric); @@ -87,40 +85,40 @@ public class JoinRender extends Renderer { reqDimension.addAll(filterDimensions); reqDimension = uniqList(reqDimension); - Set sourceMeasure = dataSource.getMeasures().stream().map(mm -> mm.getName()) + Set sourceMeasure = dataModel.getMeasures().stream().map(mm -> mm.getName()) .collect(Collectors.toSet()); - doMetric(innerSelect, filterView, queryMetrics, reqMetric, dataSource, sourceMeasure, + doMetric(innerSelect, filterView, queryMetrics, reqMetric, dataModel, sourceMeasure, scope, schema, nonAgg); - Set dimension = dataSource.getDimensions().stream().map(dd -> dd.getName()) + Set dimension = dataModel.getDimensions().stream().map(dd -> dd.getName()) .collect(Collectors.toSet()); - doDimension(innerSelect, filterDimension, queryDimension, reqDimension, dataSource, + doDimension(innerSelect, filterDimension, queryDimension, reqDimension, dataModel, dimension, scope, schema); List primary = new ArrayList<>(); - for (Identify identify : dataSource.getIdentifiers()) { + for (Identify identify : dataModel.getIdentifiers()) { primary.add(identify.getName()); if (!fieldWhere.contains(identify.getName())) { fieldWhere.add(identify.getName()); } } List dataSourceWhere = new ArrayList<>(fieldWhere); - addZipperField(dataSource, dataSourceWhere); + addZipperField(dataModel, dataSourceWhere); TableView tableView = SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension, - metricCommand.getWhere(), dataSources.get(i), scope, schema, true); + metricCommand.getWhere(), dataModels.get(i), scope, schema, true); log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString())); - String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); + String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); tableView.setAlias(alias); tableView.setPrimary(primary); - tableView.setDataSource(dataSource); + tableView.setDataModel(dataModel); if (left == null) { leftTable = tableView; left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)); - beforeSources.put(dataSource.getName(), leftTable.getAlias()); + beforeSources.put(dataModel.getName(), leftTable.getAlias()); continue; } - left = buildJoin(left, leftTable, tableView, beforeSources, dataSource, schema, scope); + left = buildJoin(left, leftTable, tableView, beforeSources, dataModel, schema, scope); leftTable = tableView; - beforeSources.put(dataSource.getName(), tableView.getAlias()); + beforeSources.put(dataModel.getName(), tableView.getAlias()); } for (Map.Entry entry : innerSelect.entrySet()) { @@ -144,16 +142,14 @@ public class JoinRender extends Renderer { } private void doMetric(Map innerSelect, TableView filterView, - List queryMetrics, List reqMetrics, DataSource dataSource, - Set sourceMeasure, SqlValidatorScope scope, SemanticSchema schema, + List queryMetrics, List reqMetrics, DataModel dataModel, + Set sourceMeasure, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { - String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); for (String m : reqMetrics) { if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) { - MetricNode metricNode = - buildMetricNode(m, dataSource, scope, schema, nonAgg, alias); + MetricNode metricNode = buildMetricNode(m, dataModel, scope, schema, nonAgg, alias); if (!metricNode.getNonAggNode().isEmpty()) { for (String measure : metricNode.getNonAggNode().keySet()) { @@ -181,14 +177,13 @@ public class JoinRender extends Renderer { } private void doDimension(Map innerSelect, Set filterDimension, - List queryDimension, List reqDimensions, DataSource dataSource, - Set dimension, SqlValidatorScope scope, SemanticSchema schema) + List queryDimension, List reqDimensions, DataModel dataModel, + Set dimension, SqlValidatorScope scope, S2CalciteSchema schema) throws Exception { - String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); for (String d : reqDimensions) { - if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) { + if (getMatchDimension(schema, dimension, dataModel, d, queryDimension)) { if (d.contains(Constants.DIMENSION_IDENTIFY)) { String[] identifyDimension = d.split(Constants.DIMENSION_IDENTIFY); innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode @@ -209,7 +204,7 @@ public class JoinRender extends Renderer { .collect(Collectors.toSet()); } - private boolean getMatchMetric(SemanticSchema schema, Set sourceMeasure, String m, + private boolean getMatchMetric(S2CalciteSchema schema, Set sourceMeasure, String m, List queryMetrics) { Optional metric = schema.getMetrics().stream() .filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst(); @@ -230,8 +225,8 @@ public class JoinRender extends Renderer { return isAdd; } - private boolean getMatchDimension(SemanticSchema schema, Set sourceDimension, - DataSource dataSource, String d, List queryDimension) { + private boolean getMatchDimension(S2CalciteSchema schema, Set sourceDimension, + DataModel dataModel, String d, List queryDimension) { String oriDimension = d; boolean isAdd = false; if (d.contains(Constants.DIMENSION_IDENTIFY)) { @@ -240,14 +235,14 @@ public class JoinRender extends Renderer { if (sourceDimension.contains(oriDimension)) { isAdd = true; } - for (Identify identify : dataSource.getIdentifiers()) { + for (Identify identify : dataModel.getIdentifiers()) { if (identify.getName().equalsIgnoreCase(oriDimension)) { isAdd = true; break; } } - if (schema.getDimension().containsKey(dataSource.getName())) { - for (Dimension dim : schema.getDimension().get(dataSource.getName())) { + if (schema.getDimensions().containsKey(dataModel.getName())) { + for (Dimension dim : schema.getDimensions().get(dataModel.getName())) { if (dim.getName().equalsIgnoreCase(oriDimension)) { isAdd = true; } @@ -264,12 +259,11 @@ public class JoinRender extends Renderer { } private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, - Map before, DataSource dataSource, SemanticSchema schema, + Map before, DataModel dataModel, S2CalciteSchema schema, SqlValidatorScope scope) throws Exception { - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); SqlNode condition = - getCondition(leftTable, tableView, dataSource, schema, scope, engineType); + getCondition(leftTable, tableView, dataModel, schema, scope, engineType); SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema); SqlNode joinRelationCondition = null; @@ -278,11 +272,11 @@ public class JoinRender extends Renderer { joinRelationCondition = getCondition(matchJoinRelation, scope, engineType); condition = joinRelationCondition; } - if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType()) + if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataModel().getTimePartType()) || Materialization.TimePartType.ZIPPER - .equals(tableView.getDataSource().getTimePartType())) { + .equals(tableView.getDataModel().getTimePartType())) { SqlNode zipperCondition = - getZipperCondition(leftTable, tableView, dataSource, schema, scope); + getZipperCondition(leftTable, tableView, dataModel, schema, scope); if (Objects.nonNull(joinRelationCondition)) { condition = new SqlBasicCall(SqlStdOperatorTable.AND, new ArrayList<>(Arrays.asList(zipperCondition, joinRelationCondition)), @@ -299,11 +293,11 @@ public class JoinRender extends Renderer { } private JoinRelation getMatchJoinRelation(Map before, TableView tableView, - SemanticSchema schema) { + S2CalciteSchema schema) { JoinRelation matchJoinRelation = JoinRelation.builder().build(); if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { for (JoinRelation joinRelation : schema.getJoinRelations()) { - if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataSource().getName()) + if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataModel().getName()) && before.containsKey(joinRelation.getLeft())) { matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() .map(r -> Triple.of( @@ -338,8 +332,8 @@ public class JoinRender extends Renderer { return condition; } - private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, - SemanticSchema schema, SqlValidatorScope scope, EngineType engineType) + private SqlNode getCondition(TableView left, TableView right, DataModel dataModel, + S2CalciteSchema schema, SqlValidatorScope scope, EngineType engineType) throws Exception { Set selectLeft = SemanticNode.getSelect(left.getTable()); @@ -347,16 +341,16 @@ public class JoinRender extends Renderer { selectLeft.retainAll(selectRight); SqlNode condition = null; for (String on : selectLeft) { - if (!SourceRender.isDimension(on, dataSource, schema)) { + if (!SourceRender.isDimension(on, dataModel, schema)) { continue; } - if (IdentifyNode.isForeign(on, left.getDataSource().getIdentifiers())) { - if (!IdentifyNode.isPrimary(on, right.getDataSource().getIdentifiers())) { + if (IdentifyNode.isForeign(on, left.getDataModel().getIdentifiers())) { + if (!IdentifyNode.isPrimary(on, right.getDataModel().getIdentifiers())) { continue; } } - if (IdentifyNode.isForeign(on, right.getDataSource().getIdentifiers())) { - if (!IdentifyNode.isPrimary(on, left.getDataSource().getIdentifiers())) { + if (IdentifyNode.isForeign(on, right.getDataModel().getIdentifiers())) { + if (!IdentifyNode.isPrimary(on, left.getDataModel().getIdentifiers())) { continue; } } @@ -396,9 +390,9 @@ public class JoinRender extends Renderer { visited.put(id, false); } - private void addZipperField(DataSource dataSource, List fields) { - if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { - dataSource.getDimensions().stream() + private void addZipperField(DataModel dataModel, List fields) { + if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) { + dataModel.getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .forEach(t -> { if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END) @@ -413,18 +407,18 @@ public class JoinRender extends Renderer { } } - private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, - SemanticSchema schema, SqlValidatorScope scope) throws Exception { - if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) + private SqlNode getZipperCondition(TableView left, TableView right, DataModel dataModel, + S2CalciteSchema schema, SqlValidatorScope scope) throws Exception { + if (Materialization.TimePartType.ZIPPER.equals(left.getDataModel().getTimePartType()) && Materialization.TimePartType.ZIPPER - .equals(right.getDataSource().getTimePartType())) { + .equals(right.getDataModel().getTimePartType())) { throw new Exception("not support two zipper table"); } SqlNode condition = null; - Optional leftTime = left.getDataSource().getDimensions().stream() + Optional leftTime = left.getDataModel().getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .findFirst(); - Optional rightTime = right.getDataSource().getDimensions().stream() + Optional rightTime = right.getDataModel().getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .findFirst(); if (leftTime.isPresent() && rightTime.isPresent()) { @@ -434,7 +428,7 @@ public class JoinRender extends Renderer { String dateTime = ""; Optional startTimeOp = (Materialization.TimePartType.ZIPPER - .equals(left.getDataSource().getTimePartType()) ? left : right).getDataSource() + .equals(left.getDataModel().getTimePartType()) ? left : right).getDataModel() .getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME .equalsIgnoreCase(d.getType())) @@ -442,7 +436,7 @@ public class JoinRender extends Renderer { .startsWith(Constants.MATERIALIZATION_ZIPPER_START)) .findFirst(); Optional endTimeOp = (Materialization.TimePartType.ZIPPER - .equals(left.getDataSource().getTimePartType()) ? left : right).getDataSource() + .equals(left.getDataModel().getTimePartType()) ? left : right).getDataModel() .getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME .equalsIgnoreCase(d.getType())) @@ -451,17 +445,17 @@ public class JoinRender extends Renderer { .findFirst(); if (startTimeOp.isPresent() && endTimeOp.isPresent()) { TableView zipper = Materialization.TimePartType.ZIPPER - .equals(left.getDataSource().getTimePartType()) ? left : right; + .equals(left.getDataModel().getTimePartType()) ? left : right; TableView partMetric = Materialization.TimePartType.ZIPPER - .equals(left.getDataSource().getTimePartType()) ? right : left; + .equals(left.getDataModel().getTimePartType()) ? right : left; Optional partTime = Materialization.TimePartType.ZIPPER - .equals(left.getDataSource().getTimePartType()) ? rightTime : leftTime; + .equals(left.getDataModel().getTimePartType()) ? rightTime : leftTime; startTime = zipper.getAlias() + "." + startTimeOp.get().getName(); endTime = zipper.getAlias() + "." + endTimeOp.get().getName(); dateTime = partMetric.getAlias() + "." + partTime.get().getName(); } EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType.fromString(schema.getOntology().getDatabase().getType()); ArrayList operandList = new ArrayList<>(Arrays.asList(SemanticNode.parse(endTime, scope, engineType), SemanticNode.parse(dateTime, scope, engineType))); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java index 92dffeb9d..ef20426f9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java @@ -3,9 +3,8 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; @@ -23,11 +22,10 @@ import java.util.List; public class OutputRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataSources, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { + public void render(MetricQueryParam metricCommand, List dataModels, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView selectDataSet = super.tableView; - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); for (String dimension : metricCommand.getDimensions()) { selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope, engineType)); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java similarity index 91% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java index 088a98e99..d4322a411 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java @@ -1,13 +1,14 @@ -package com.tencent.supersonic.headless.core.translator.calcite.sql; +package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; @@ -27,29 +28,29 @@ public abstract class Renderer { protected TableView tableView = new TableView(); - public static Optional getDimensionByName(String name, DataSource datasource) { + public static Optional getDimensionByName(String name, DataModel datasource) { return datasource.getDimensions().stream().filter(d -> d.getName().equalsIgnoreCase(name)) .findFirst(); } - public static Optional getMeasureByName(String name, DataSource datasource) { + public static Optional getMeasureByName(String name, DataModel datasource) { return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name)) .findFirst(); } - public static Optional getMetricByName(String name, SemanticSchema schema) { + public static Optional getMetricByName(String name, S2CalciteSchema schema) { Optional metric = schema.getMetrics().stream() .filter(m -> m.getName().equalsIgnoreCase(name)).findFirst(); return metric; } - public static Optional getIdentifyByName(String name, DataSource datasource) { + public static Optional getIdentifyByName(String name, DataModel datasource) { return datasource.getIdentifiers().stream().filter(i -> i.getName().equalsIgnoreCase(name)) .findFirst(); } - public static MetricNode buildMetricNode(String metric, DataSource datasource, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg, String alias) + public static MetricNode buildMetricNode(String metric, DataModel datasource, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg, String alias) throws Exception { Optional metricOpt = getMetricByName(metric, schema); MetricNode metricNode = new MetricNode(); @@ -113,6 +114,6 @@ public abstract class Renderer { return SemanticNode.buildAs(alias, tableView.build()); } - public abstract void render(MetricQueryParam metricCommand, List dataSources, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception; + public abstract void render(MetricQueryParam metricCommand, List dataModels, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java index 140d601d9..c4eb0d24c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java @@ -3,16 +3,15 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; -import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; -import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; +import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DimensionNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.IdentifyNode; @@ -21,7 +20,6 @@ import com.tencent.supersonic.headless.core.translator.calcite.sql.node.Semantic import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.apache.calcite.util.Litmus; import org.springframework.util.CollectionUtils; import java.util.ArrayList; @@ -44,7 +42,7 @@ public class SourceRender extends Renderer { public static TableView renderOne(String alias, List fieldWheres, List reqMetrics, List reqDimensions, String queryWhere, - DataSource datasource, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) + DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView dataSet = new TableView(); @@ -97,7 +95,7 @@ public class SourceRender extends Renderer { output.setMeasure(SemanticNode.deduplicateNode(output.getMeasure())); dataSet.setMeasure(SemanticNode.deduplicateNode(dataSet.getMeasure())); - SqlNode tableNode = DataSourceNode.buildExtend(datasource, extendFields, scope); + SqlNode tableNode = DataModelNode.buildExtend(datasource, extendFields, scope); dataSet.setTable(tableNode); output.setTable( SemanticNode.buildAs(Constants.DATASOURCE_TABLE_OUT_PREFIX + datasource.getName() @@ -107,12 +105,11 @@ public class SourceRender extends Renderer { - private static void buildDimension(String alias, String dimension, DataSource datasource, - SemanticSchema schema, boolean nonAgg, Map extendFields, + private static void buildDimension(String alias, String dimension, DataModel datasource, + S2CalciteSchema schema, boolean nonAgg, Map extendFields, TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception { - List dimensionList = schema.getDimension().get(datasource.getName()); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + List dimensionList = schema.getDimensions().get(datasource.getName()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); boolean isAdd = false; if (!CollectionUtils.isEmpty(dimensionList)) { for (Dimension dim : dimensionList) { @@ -186,12 +183,11 @@ public class SourceRender extends Renderer { } private static List getWhereMeasure(List fields, List queryMetrics, - List queryDimensions, Map extendFields, DataSource datasource, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { + List queryDimensions, Map extendFields, DataModel datasource, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { Iterator iterator = fields.iterator(); List whereNode = new ArrayList<>(); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); while (iterator.hasNext()) { String cur = iterator.next(); if (queryDimensions.contains(cur) || queryMetrics.contains(cur)) { @@ -199,7 +195,7 @@ public class SourceRender extends Renderer { } } for (String where : fields) { - List dimensionList = schema.getDimension().get(datasource.getName()); + List dimensionList = schema.getDimensions().get(datasource.getName()); boolean isAdd = false; if (!CollectionUtils.isEmpty(dimensionList)) { for (Dimension dim : dimensionList) { @@ -229,8 +225,8 @@ public class SourceRender extends Renderer { private static void mergeWhere(List fields, TableView dataSet, TableView outputSet, List queryMetrics, List queryDimensions, - Map extendFields, DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + Map extendFields, DataModel datasource, SqlValidatorScope scope, + S2CalciteSchema schema, boolean nonAgg) throws Exception { List whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource, scope, schema, nonAgg); dataSet.getMeasure().addAll(whereNode); @@ -238,7 +234,7 @@ public class SourceRender extends Renderer { } public static void whereDimMetric(List fields, List queryMetrics, - List queryDimensions, DataSource datasource, SemanticSchema schema, + List queryDimensions, DataModel datasource, S2CalciteSchema schema, Set dimensions, Set metrics) { for (String field : fields) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) { @@ -252,8 +248,8 @@ public class SourceRender extends Renderer { } } - private static void addField(String field, String oriField, DataSource datasource, - SemanticSchema schema, Set dimensions, Set metrics) { + private static void addField(String field, String oriField, DataModel datasource, + S2CalciteSchema schema, Set dimensions, Set metrics) { Optional dimension = datasource.getDimensions().stream() .filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); if (dimension.isPresent()) { @@ -266,8 +262,8 @@ public class SourceRender extends Renderer { dimensions.add(oriField); return; } - if (schema.getDimension().containsKey(datasource.getName())) { - Optional dataSourceDim = schema.getDimension().get(datasource.getName()) + if (schema.getDimensions().containsKey(datasource.getName())) { + Optional dataSourceDim = schema.getDimensions().get(datasource.getName()) .stream().filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); if (dataSourceDim.isPresent()) { dimensions.add(oriField); @@ -293,7 +289,7 @@ public class SourceRender extends Renderer { } } - public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) { + public static boolean isDimension(String name, DataModel datasource, S2CalciteSchema schema) { Optional dimension = datasource.getDimensions().stream() .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); if (dimension.isPresent()) { @@ -304,8 +300,8 @@ public class SourceRender extends Renderer { if (identify.isPresent()) { return true; } - if (schema.getDimension().containsKey(datasource.getName())) { - Optional dataSourceDim = schema.getDimension().get(datasource.getName()) + if (schema.getDimensions().containsKey(datasource.getName())) { + Optional dataSourceDim = schema.getDimensions().get(datasource.getName()) .stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); if (dataSourceDim.isPresent()) { return true; @@ -314,13 +310,13 @@ public class SourceRender extends Renderer { return false; } - private static void addTimeDimension(DataSource dataSource, List queryDimension) { - if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { - Optional startTimeOp = dataSource.getDimensions().stream() + private static void addTimeDimension(DataModel dataModel, List queryDimension) { + if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) { + Optional startTimeOp = dataModel.getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START)) .findFirst(); - Optional endTimeOp = dataSource.getDimensions().stream() + Optional endTimeOp = dataModel.getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)) .findFirst(); @@ -331,7 +327,7 @@ public class SourceRender extends Renderer { queryDimension.add(endTimeOp.get().getName()); } } else { - Optional timeOp = dataSource.getDimensions().stream() + Optional timeOp = dataModel.getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .findFirst(); if (timeOp.isPresent() && !queryDimension.contains(timeOp.get().getName())) { @@ -340,27 +336,26 @@ public class SourceRender extends Renderer { } } - public void render(MetricQueryParam metricQueryParam, List dataSources, - SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { + public void render(MetricQueryParam metricQueryParam, List dataModels, + SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricQueryParam.getWhere(); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); - EngineType engineType = - EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); if (queryWhere != null && !queryWhere.isEmpty()) { SqlNode sqlNode = SemanticNode.parse(queryWhere, scope, engineType); FilterNode.getFilterField(sqlNode, whereFields); fieldWhere = whereFields.stream().collect(Collectors.toList()); } - if (dataSources.size() == 1) { - DataSource dataSource = dataSources.get(0); + if (dataModels.size() == 1) { + DataModel dataModel = dataModels.get(0); super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(), - metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataSource, - scope, schema, nonAgg); + metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataModel, scope, + schema, nonAgg); return; } JoinRender joinRender = new JoinRender(); - joinRender.render(metricQueryParam, dataSources, scope, schema, nonAgg); + joinRender.render(metricQueryParam, dataModels, scope, schema, nonAgg); super.tableView = joinRender.getTableView(); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java index 252dd3184..e0287e427 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java @@ -97,7 +97,7 @@ public class CalculateAggConverter implements QueryConverter { @Override public void convert(QueryStatement queryStatement) throws Exception { - Database database = queryStatement.getSemanticModel().getDatabase(); + Database database = queryStatement.getOntology().getDatabase(); DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement, EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); queryStatement.setDataSetQueryParam(dataSetQueryParam); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java index b06e363f4..54b24dd84 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java @@ -34,7 +34,7 @@ public class DefaultDimValueConverter implements QueryConverter { @Override public void convert(QueryStatement queryStatement) { - List dimensions = queryStatement.getSemanticModel().getDimensions().stream() + List dimensions = queryStatement.getOntology().getDimensions().stream() .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dimensions)) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java index 33102db14..12c8dd722 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -60,7 +60,7 @@ public class ParserDefaultConverter implements QueryConverter { // support detail query if (queryParam.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) { - Map modelMap = queryStatement.getSemanticModel().getModelMap(); + Map modelMap = queryStatement.getOntology().getModelMap(); for (Long modelId : modelMap.keySet()) { String modelBizName = modelMap.get(modelId).getName(); String internalMetricName = diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java index 179ead737..e2c5f9a06 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java @@ -4,7 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -39,9 +39,9 @@ public class SqlVariableParseConverter implements QueryConverter { SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(), modelResp.getModelDetail().getSqlVariables(), queryStatement.getQueryParam().getParams()); - DataSource dataSource = queryStatement.getSemanticModel().getDatasourceMap() - .get(modelResp.getBizName()); - dataSource.setSqlQuery(sqlParsed); + DataModel dataModel = + queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName()); + dataModel.setSqlQuery(sqlParsed); } } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index fdd2750c0..2cb623acc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -260,9 +260,8 @@ public class S2DataPermissionAspect { } public void checkModelVisible(User user, Set modelIds) { - List modelListVisible = - modelService.getModelListWithAuth(user, null, AuthType.VISIBLE).stream() - .map(ModelResp::getId).collect(Collectors.toList()); + List modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VIEWER) + .stream().map(ModelResp::getId).collect(Collectors.toList()); List modelIdCopied = new ArrayList<>(modelIds); modelIdCopied.removeAll(modelListVisible); if (!CollectionUtils.isEmpty(modelIdCopied)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index f1b43a77d..c26b66c44 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -34,7 +34,6 @@ import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.SemanticTranslator; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.server.annotation.S2DataPermission; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; @@ -44,13 +43,13 @@ import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker; -import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -67,7 +66,6 @@ public class S2SemanticLayerService implements SemanticLayerService { private final StatUtils statUtils; private final QueryUtils queryUtils; - private final QueryReqConverter queryReqConverter; private final SemanticSchemaManager semanticSchemaManager; private final DataSetService dataSetService; private final SchemaService schemaService; @@ -80,14 +78,13 @@ public class S2SemanticLayerService implements SemanticLayerService { private final List queryExecutors = ComponentFactory.getQueryExecutors(); public S2SemanticLayerService(StatUtils statUtils, QueryUtils queryUtils, - QueryReqConverter queryReqConverter, SemanticSchemaManager semanticSchemaManager, - DataSetService dataSetService, SchemaService schemaService, - SemanticTranslator semanticTranslator, MetricDrillDownChecker metricDrillDownChecker, + SemanticSchemaManager semanticSchemaManager, DataSetService dataSetService, + SchemaService schemaService, SemanticTranslator semanticTranslator, + MetricDrillDownChecker metricDrillDownChecker, KnowledgeBaseService knowledgeBaseService, MetricService metricService, DimensionService dimensionService) { this.statUtils = statUtils; this.queryUtils = queryUtils; - this.queryReqConverter = queryReqConverter; this.semanticSchemaManager = semanticSchemaManager; this.dataSetService = dataSetService; this.schemaService = schemaService; @@ -122,7 +119,6 @@ public class S2SemanticLayerService implements SemanticLayerService { statUtils.initStatInfo(queryReq, user); // 2.query from cache - String cacheKey = queryCache.getCacheKey(queryReq); Object query = queryCache.query(queryReq, cacheKey); if (Objects.nonNull(query)) { @@ -136,16 +132,16 @@ public class S2SemanticLayerService implements SemanticLayerService { } StatUtils.get().setUseResultCache(false); - // 3 query + // 3 translate query QueryStatement queryStatement = buildQueryStatement(queryReq, user); + semanticTranslator.translate(queryStatement); + + // Check whether the dimensions of the metric drill-down are correct temporarily, + // add the abstraction of a validator later. + metricDrillDownChecker.checkQuery(queryStatement); + + // 4.execute query SemanticQueryResp queryResp = null; - - // skip translation if already done. - if (!queryStatement.isTranslated()) { - semanticTranslator.translate(queryStatement); - } - queryPreCheck(queryStatement); - for (QueryExecutor queryExecutor : queryExecutors) { if (queryExecutor.accept(queryStatement)) { queryResp = queryExecutor.execute(queryStatement); @@ -154,7 +150,7 @@ public class S2SemanticLayerService implements SemanticLayerService { } } - // 4 reset cache and set stateInfo + // 5.reset cache and set stateInfo Boolean setCacheSuccess = queryCache.put(cacheKey, queryResp); if (setCacheSuccess) { // if result is not null, update cache data @@ -185,7 +181,7 @@ public class S2SemanticLayerService implements SemanticLayerService { List dimensionValues = getDimensionValuesFromDict(dimensionValueReq, dataSetIds); - // If the search results are null, search dimensionValue from the database + // try to query dimensionValue from the database. if (CollectionUtils.isEmpty(dimensionValues)) { return getDimensionValuesFromDb(dimensionValueReq, user); } @@ -218,9 +214,29 @@ public class S2SemanticLayerService implements SemanticLayerService { .map(MapResult::getName).collect(Collectors.toList()); } - private SemanticQueryResp getDimensionValuesFromDb(DimensionValueReq dimensionValueReq, + private SemanticQueryResp getDimensionValuesFromDb(DimensionValueReq queryDimValueReq, User user) { - QuerySqlReq querySqlReq = buildQuerySqlReq(dimensionValueReq); + QuerySqlReq querySqlReq = new QuerySqlReq(); + List modelResps = + schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); + DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getBizName(), + queryDimValueReq.getModelId()); + ModelResp modelResp = modelResps.get(0); + String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(), + modelResp.getName()); + List timeDims = modelResp.getTimeDimension(); + if (CollectionUtils.isNotEmpty(timeDims)) { + sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, + TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(), + TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate()); + } + if (StringUtils.isNotBlank(queryDimValueReq.getValue())) { + sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%" + + queryDimValueReq.getValue() + "%'"; + } + querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); + querySqlReq.setSql(sql); + return queryByReq(querySqlReq, user); } @@ -271,35 +287,16 @@ public class S2SemanticLayerService implements SemanticLayerService { return metricService.getMetrics(metaFilter); } - private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) - throws Exception { - // If dataSetId or DataSetName is empty, parse dataSetId from the SQL - if (querySqlReq.needGetDataSetId()) { - Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user); - querySqlReq.setDataSetId(dataSetId); - } - SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq); - SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); - QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, semanticSchemaResp); - queryStatement.setModelIds(querySqlReq.getModelIds()); - queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - queryStatement.setSemanticSchemaResp(semanticSchemaResp); - queryStatement.setSemanticModel(semanticSchemaManager.getSemanticModel(semanticSchemaResp)); - return queryStatement; - } - - private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) - throws Exception { + private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) { QueryStatement queryStatement = null; if (semanticQueryReq instanceof QuerySqlReq) { queryStatement = buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user); } if (semanticQueryReq instanceof QueryStructReq) { - queryStatement = buildStructQueryStatement((QueryStructReq) semanticQueryReq); + queryStatement = buildStructQueryStatement(semanticQueryReq); } if (semanticQueryReq instanceof QueryMultiStructReq) { - queryStatement = - buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user); + queryStatement = buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); } if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) { @@ -310,72 +307,46 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryStatement; } - private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) { - SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq); - SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); - QueryStatement queryStatement = new QueryStatement(); - QueryParam queryParam = new QueryParam(); - queryReqConverter.convert(queryStructReq, queryParam); - queryStatement.setQueryParam(queryParam); - queryStatement.setIsS2SQL(false); - queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - queryStatement.setDataSetId(queryStructReq.getDataSetId()); - queryStatement.setSemanticSchemaResp(semanticSchemaResp); - queryStatement.setSemanticModel(semanticSchemaManager.getSemanticModel(semanticSchemaResp)); + private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) { + // If dataSetId or DataSetName is empty, parse dataSetId from the SQL + if (querySqlReq.needGetDataSetId()) { + Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user); + querySqlReq.setDataSetId(dataSetId); + } + + QueryStatement queryStatement = buildStructQueryStatement(querySqlReq); + queryStatement.setIsS2SQL(true); + queryStatement.setSql(querySqlReq.getSql()); return queryStatement; } - private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq, - User user) throws Exception { - List sqlParsers = new ArrayList<>(); - for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { - QueryStatement queryStatement = buildQueryStatement(queryStructReq, user); - SemanticModel semanticModel = queryStatement.getSemanticModel(); - queryStatement.setModelIds(queryStructReq.getModelIds()); - queryStatement.setSemanticModel(semanticModel); - queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - semanticTranslator.translate(queryStatement); - sqlParsers.add(queryStatement); - } - log.info("multi sqlParser:{}", sqlParsers); - return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); - } - - private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { + private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); - schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId()); - schemaFilterReq.setModelIds(semanticQueryReq.getModelIds()); - return schemaFilterReq; + schemaFilterReq.setDataSetId(queryReq.getDataSetId()); + schemaFilterReq.setModelIds(queryReq.getModelIds()); + SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(schemaFilterReq); + + QueryStatement queryStatement = new QueryStatement(); + QueryParam queryParam = new QueryParam(); + BeanUtils.copyProperties(queryReq, queryParam); + queryStatement.setQueryParam(queryParam); + queryStatement.setModelIds(queryReq.getModelIds()); + queryStatement.setEnableOptimize(queryUtils.enableOptimize()); + queryStatement.setDataSetId(queryReq.getDataSetId()); + queryStatement.setSemanticSchemaResp(semanticSchemaResp); + queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp)); + return queryStatement; } - private QuerySqlReq buildQuerySqlReq(DimensionValueReq queryDimValueReq) { - QuerySqlReq querySqlReq = new QuerySqlReq(); - List modelResps = - schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); - DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getBizName(), - queryDimValueReq.getModelId()); - ModelResp modelResp = modelResps.get(0); - String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(), - modelResp.getName()); - List timeDims = modelResp.getTimeDimension(); - if (CollectionUtils.isNotEmpty(timeDims)) { - sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, - TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(), - TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate()); + private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq) { + List queryStatements = new ArrayList<>(); + for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { + QueryStatement queryStatement = buildStructQueryStatement(queryStructReq); + semanticTranslator.translate(queryStatement); + queryStatements.add(queryStatement); } - if (StringUtils.isNotBlank(queryDimValueReq.getValue())) { - sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%" - + queryDimValueReq.getValue() + "%'"; - } - querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); - querySqlReq.setSql(sql); - return querySqlReq; - } - - private void queryPreCheck(QueryStatement queryStatement) { - // Check whether the dimensions of the metric drill-down are correct temporarily, - // add the abstraction of a validator later. - metricDrillDownChecker.checkQuery(queryStatement); + log.info("Union multiple query statements:{}", queryStatements); + return queryUtils.unionAll(queryMultiStructReq, queryStatements); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 628a3b79d..7e5eff936 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DimensionTimeTypeParams; @@ -18,8 +18,8 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materializa import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; @@ -57,9 +57,8 @@ public class SemanticSchemaManager { this.schemaService = schemaService; } - public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) { - SemanticModel semanticModel = new SemanticModel(); - semanticModel.setSchemaKey(semanticSchemaResp.getSchemaKey()); + public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) { + Ontology ontology = new Ontology(); Map> dimensionYamlTpls = new HashMap<>(); List dataModelYamlTpls = new ArrayList<>(); List metricYamlTpls = new ArrayList<>(); @@ -67,36 +66,35 @@ public class SemanticSchemaManager { schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp(); - semanticModel.setDatabase(DatabaseConverter.convert(databaseResp)); + ontology.setDatabase(DatabaseConverter.convert(databaseResp)); if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) { - semanticModel.setJoinRelations( + ontology.setJoinRelations( getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { - Map dataSourceMap = - dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect( - Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); - semanticModel.setDatasourceMap(dataSourceMap); + Map dataModelMap = + dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect( + Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1)); + ontology.setDataModelMap(dataModelMap); } if (!dimensionYamlTpls.isEmpty()) { Map> dimensionMap = new HashMap<>(); for (Map.Entry> entry : dimensionYamlTpls.entrySet()) { dimensionMap.put(entry.getKey(), getDimensions(entry.getValue())); } - semanticModel.setDimensionMap(dimensionMap); + ontology.setDimensionMap(dimensionMap); } if (!metricYamlTpls.isEmpty()) { - semanticModel.setMetrics(getMetrics(metricYamlTpls)); + ontology.setMetrics(getMetrics(metricYamlTpls)); } - return semanticModel; + return ontology; } - public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) - throws Exception { + public Ontology getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception { if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) { throw new Exception("semanticSchemaResp tag is empty"); } - SemanticModel semanticModel = getSemanticModel(semanticSchemaResp); + Ontology ontology = buildOntology(semanticSchemaResp); // Map> dimensions = new HashMap<>(); Map> tagMap = new HashMap<>(); for (TagResp tagResp : semanticSchemaResp.getTags()) { @@ -105,25 +103,23 @@ public class SemanticSchemaManager { } tagMap.get(tagResp.getModelId()).add(tagResp); } - if (Objects.nonNull(semanticModel.getDatasourceMap()) - && !semanticModel.getDatasourceMap().isEmpty()) { - for (Map.Entry entry : semanticModel.getDatasourceMap() - .entrySet()) { + if (Objects.nonNull(ontology.getDataModelMap()) && !ontology.getDataModelMap().isEmpty()) { + for (Map.Entry entry : ontology.getDataModelMap().entrySet()) { List modelDimensions = new ArrayList<>(); - if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) { - semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions); + if (!ontology.getDimensionMap().containsKey(entry.getKey())) { + ontology.getDimensionMap().put(entry.getKey(), modelDimensions); } else { - modelDimensions = semanticModel.getDimensionMap().get(entry.getKey()); + modelDimensions = ontology.getDimensionMap().get(entry.getKey()); } if (tagMap.containsKey(entry.getValue().getId())) { for (TagResp tagResp : tagMap.get(entry.getValue().getId())) { - addTagModel(tagResp, modelDimensions, semanticModel.getMetrics()); + addTagModel(tagResp, modelDimensions, ontology.getMetrics()); } } } } - return semanticModel; + return ontology; } private void addTagModel(TagResp tagResp, List modelDimensions, @@ -178,30 +174,30 @@ public class SemanticSchemaManager { return getDimension(t); } - public static DataSource getDatasource(final DataModelYamlTpl d) { - DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId()) + public static DataModel getDataModel(final DataModelYamlTpl d) { + DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId()) .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()) .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) .measures(getMeasureParams(d.getMeasures())) .dimensions(getDimensions(d.getDimensions())).build(); - datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions())); + dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions())); if (Objects.nonNull(d.getModelSourceTypeEnum())) { - datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); + dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); } if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) { - Set measures = datasource.getMeasures().stream().map(mm -> mm.getName()) + Set measures = dataModel.getMeasures().stream().map(mm -> mm.getName()) .collect(Collectors.toSet()); for (Field f : d.getFields()) { if (!measures.contains(f.getFieldName())) { - datasource.getMeasures().add(Measure.builder().expr(f.getFieldName()) + dataModel.getMeasures().add(Measure.builder().expr(f.getFieldName()) .name(f.getFieldName()).agg("").build()); } } } - return datasource; + return dataModel; } - private static String getDataSourceAggTime(List dimensions) { + private static String getDataModelAggTime(List dimensions) { Optional timeDimension = dimensions.stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .findFirst(); @@ -356,39 +352,39 @@ public class SemanticSchemaManager { return joinRelations; } - public static void update(SemanticSchema schema, List metric) throws Exception { + public static void update(S2CalciteSchema schema, List metric) throws Exception { if (schema != null) { updateMetric(metric, schema.getMetrics()); } } - public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) + public static void update(S2CalciteSchema schema, DataModel datasourceYamlTpl) throws Exception { if (schema != null) { String dataSourceName = datasourceYamlTpl.getName(); - Optional> datasourceYamlTplMap = - schema.getDatasource().entrySet().stream() + Optional> datasourceYamlTplMap = + schema.getDataModels().entrySet().stream() .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst(); if (datasourceYamlTplMap.isPresent()) { datasourceYamlTplMap.get().setValue(datasourceYamlTpl); } else { - schema.getDatasource().put(dataSourceName, datasourceYamlTpl); + schema.getDataModels().put(dataSourceName, datasourceYamlTpl); } } } - public static void update(SemanticSchema schema, String datasourceBizName, + public static void update(S2CalciteSchema schema, String datasourceBizName, List dimensionYamlTpls) throws Exception { if (schema != null) { Optional>> datasourceYamlTplMap = schema - .getDimension().entrySet().stream() + .getDimensions().entrySet().stream() .filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst(); if (datasourceYamlTplMap.isPresent()) { updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue()); } else { List dimensions = new ArrayList<>(); updateDimension(dimensionYamlTpls, dimensions); - schema.getDimension().put(datasourceBizName, dimensions); + schema.getDimensions().put(datasourceBizName, dimensions); } } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java index fd9f69d43..be75ac482 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java @@ -126,7 +126,7 @@ public class DomainServiceImpl implements DomainService { return domainWithAuth.stream().peek(domainResp -> domainResp.setHasEditPermission(true)) .collect(Collectors.toSet()); } - if (authTypeEnum.equals(AuthType.VISIBLE)) { + if (authTypeEnum.equals(AuthType.VIEWER)) { domainWithAuth = domainResps.stream() .filter(domainResp -> checkViewPermission(orgIds, user, domainResp)) .collect(Collectors.toSet()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index f59d7e64e..e1c64a445 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -428,7 +428,7 @@ public class ModelServiceImpl implements ModelService { .filter(modelResp -> checkAdminPermission(orgIds, user, modelResp)) .collect(Collectors.toList()); } - if (authTypeEnum.equals(AuthType.VISIBLE)) { + if (authTypeEnum.equals(AuthType.VIEWER)) { modelWithAuth = modelResps.stream() .filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp)) .collect(Collectors.toList()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java index 74470a188..12caf5b77 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java @@ -32,7 +32,7 @@ public class MetricDrillDownChecker { public void checkQuery(QueryStatement queryStatement) { SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - String sql = queryStatement.getDataSetQueryParam().getSql(); + String sql = queryStatement.getSql(); if (StringUtils.isBlank(sql)) { return; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java deleted file mode 100644 index 3236356f5..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ /dev/null @@ -1,376 +0,0 @@ -package com.tencent.supersonic.headless.server.utils; - -import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; -import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.MetricTable; -import com.tencent.supersonic.headless.api.pojo.QueryParam; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.api.pojo.enums.MetricType; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; -import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Component -@Slf4j -public class QueryReqConverter { - - @Autowired - private QueryStructUtils queryStructUtils; - - @Autowired - private SqlGenerateUtils sqlGenerateUtils; - - public QueryStatement convert(QuerySqlReq querySQLReq, SemanticSchemaResp semanticSchemaResp) - throws Exception { - - if (semanticSchemaResp == null) { - return new QueryStatement(); - } - // 1.convert name to bizName - convertNameToBizName(querySQLReq, semanticSchemaResp); - // 2.functionName corrector - functionNameCorrector(querySQLReq, semanticSchemaResp); - // 3.correct tableName - correctTableName(querySQLReq); - // 4.remove Underscores - querySQLReq.setSql(SqlRemoveHelper.removeUnderscores(querySQLReq.getSql())); - - String tableName = SqlSelectHelper.getTableName(querySQLReq.getSql()); - if (StringUtils.isEmpty(tableName)) { - return new QueryStatement(); - } - // correct order item is same as agg alias - String reqSql = querySQLReq.getSql(); - querySQLReq.setSql(SqlReplaceHelper.replaceAggAliasOrderItem(querySQLReq.getSql())); - log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, querySQLReq.getSql()); - // 5.build MetricTables - List allFields = SqlSelectHelper.getAllSelectFields(querySQLReq.getSql()); - List metricSchemas = getMetrics(semanticSchemaResp, allFields); - List metrics = - metricSchemas.stream().map(m -> m.getBizName()).collect(Collectors.toList()); - QueryStructReq queryStructReq = new QueryStructReq(); - MetricTable metricTable = new MetricTable(); - metricTable.setMetrics(metrics); - - Set dimensions = getDimensions(semanticSchemaResp, allFields); - - metricTable.setDimensions(new ArrayList<>(dimensions)); - - metricTable.setAlias(tableName.toLowerCase()); - // if metric empty , fill model default - if (CollectionUtils.isEmpty(metricTable.getMetrics())) { - metricTable.setMetrics(new ArrayList<>()); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); - } else { - queryStructReq.setAggregators(metricTable.getMetrics().stream() - .map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)) - .collect(Collectors.toList())); - } - AggOption aggOption = getAggOption(querySQLReq, metricSchemas); - metricTable.setAggOption(aggOption); - List tables = new ArrayList<>(); - tables.add(metricTable); - // 6.build ParseSqlReq - DataSetQueryParam result = new DataSetQueryParam(); - BeanUtils.copyProperties(querySQLReq, result); - - result.setTables(tables); - DatabaseResp database = semanticSchemaResp.getDatabaseResp(); - if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), - database.getVersion())) { - result.setSupportWith(false); - result.setWithAlias(false); - } - // 7. do deriveMetric - generateDerivedMetric(semanticSchemaResp, aggOption, result); - // 8.physicalSql by ParseSqlReq - - queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); - queryStructReq.setDataSetId(querySQLReq.getDataSetId()); - queryStructReq.setQueryType(getQueryType(aggOption)); - log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq); - QueryParam queryParam = new QueryParam(); - convert(queryStructReq, queryParam); - QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryParam(queryParam); - queryStatement.setDataSetQueryParam(result); - queryStatement.setIsS2SQL(true); - queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - queryStatement.setDataSetId(querySQLReq.getDataSetId()); - queryStatement.setLimit(querySQLReq.getLimit()); - - return queryStatement; - } - - public void convert(QueryStructReq queryStructReq, QueryParam queryParam) { - BeanUtils.copyProperties(queryStructReq, queryParam); - queryParam.setOrders(queryStructReq.getOrders()); - queryParam.setMetrics(queryStructReq.getMetrics()); - queryParam.setGroups(queryStructReq.getGroups()); - } - - private AggOption getAggOption(QuerySqlReq databaseReq, List metricSchemas) { - String sql = databaseReq.getSql(); - if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) - && !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) { - log.debug("getAggOption simple sql set to DEFAULT"); - return AggOption.DEFAULT; - } - // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" - // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" - if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) - || SqlSelectFunctionHelper.hasFunction(sql, "count") - || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { - return AggOption.OUTER; - } - if (databaseReq.isInnerLayerNative()) { - return AggOption.NATIVE; - } - if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) - || SqlSelectHelper.hasGroupBy(sql)) { - return AggOption.OUTER; - } - long defaultAggNullCnt = metricSchemas.stream().filter( - m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg())) - .count(); - if (defaultAggNullCnt > 0) { - log.debug("getAggOption find null defaultAgg metric set to NATIVE"); - return AggOption.OUTER; - } - return AggOption.DEFAULT; - } - - private void convertNameToBizName(QuerySqlReq querySqlReq, - SemanticSchemaResp semanticSchemaResp) { - Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); - String sql = querySqlReq.getSql(); - log.debug("dataSetId:{},convert name to bizName before:{}", querySqlReq.getDataSetId(), - sql); - sql = SqlReplaceHelper.replaceSqlByPositions(sql); - log.debug("replaceSqlByPositions:{}", sql); - String replaceFields = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); - log.debug("dataSetId:{},convert name to bizName after:{}", querySqlReq.getDataSetId(), - replaceFields); - querySqlReq.setSql(replaceFields); - } - - private Set getDimensions(SemanticSchemaResp semanticSchemaResp, - List allFields) { - Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() - .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), - SchemaItem::getBizName, (k1, k2) -> k1)); - Map internalLowerToNameMap = QueryStructUtils.internalCols.stream() - .collect(Collectors.toMap(String::toLowerCase, a -> a)); - dimensionLowerToNameMap.putAll(internalLowerToNameMap); - return allFields.stream() - .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) - .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())) - .collect(Collectors.toSet()); - } - - private List getMetrics(SemanticSchemaResp semanticSchemaResp, - List allFields) { - Map metricLowerToNameMap = - semanticSchemaResp.getMetrics().stream().collect(Collectors - .toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); - return allFields.stream() - .filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) - .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())) - .collect(Collectors.toList()); - } - - private void functionNameCorrector(QuerySqlReq databaseReq, - SemanticSchemaResp semanticSchemaResp) { - DatabaseResp database = semanticSchemaResp.getDatabaseResp(); - if (Objects.isNull(database) || Objects.isNull(database.getType())) { - return; - } - String type = database.getType(); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase()); - if (Objects.nonNull(engineAdaptor)) { - String functionNameCorrector = - engineAdaptor.functionNameCorrector(databaseReq.getSql()); - databaseReq.setSql(functionNameCorrector); - } - } - - protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { - // support fieldName and field alias to bizName - Map dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap( - entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - - Map metricResults = semanticSchemaResp.getMetrics().stream().flatMap( - entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - - dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); - dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); - dimensionResults.putAll(metricResults); - return dimensionResults; - } - - private Stream> getPairStream(String aliasStr, String name, - String bizName) { - Set> elements = new HashSet<>(); - elements.add(Pair.of(name, bizName)); - if (StringUtils.isNotBlank(aliasStr)) { - List aliasList = SchemaItem.getAliasList(aliasStr); - for (String alias : aliasList) { - elements.add(Pair.of(alias, bizName)); - } - } - return elements.stream(); - } - - public void correctTableName(QuerySqlReq querySqlReq) { - String sql = querySqlReq.getSql(); - sql = SqlReplaceHelper.replaceTable(sql, - Constants.TABLE_PREFIX + querySqlReq.getDataSetId()); - log.debug("correctTableName after:{}", sql); - querySqlReq.setSql(sql); - } - - private QueryType getQueryType(AggOption aggOption) { - boolean isAgg = AggOption.isAgg(aggOption); - QueryType queryType = QueryType.DETAIL; - if (isAgg) { - queryType = QueryType.AGGREGATE; - } - return queryType; - } - - private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, - DataSetQueryParam viewQueryParam) { - String sql = viewQueryParam.getSql(); - for (MetricTable metricTable : viewQueryParam.getTables()) { - Set measures = new HashSet<>(); - Map replaces = generateDerivedMetric(semanticSchemaResp, aggOption, - metricTable.getMetrics(), metricTable.getDimensions(), measures); - - if (!CollectionUtils.isEmpty(replaces)) { - // metricTable sql use measures replace metric - sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces); - metricTable.setAggOption(AggOption.NATIVE); - // metricTable use measures replace metric - if (!CollectionUtils.isEmpty(measures)) { - metricTable.setMetrics(new ArrayList<>(measures)); - } else { - // empty measure , fill default - metricTable.setMetrics(new ArrayList<>()); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); - } - } - } - viewQueryParam.setSql(sql); - } - - private Map generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, - AggOption aggOption, List metrics, List dimensions, - Set measures) { - Map result = new HashMap<>(); - List metricResps = semanticSchemaResp.getMetrics(); - List dimensionResps = semanticSchemaResp.getDimensions(); - - // Check if any metric is derived - boolean hasDerivedMetrics = - metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType - .isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams())); - if (!hasDerivedMetrics) { - return result; - } - - log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics); - - Set allFields = new HashSet<>(); - Map allMeasures = new HashMap<>(); - semanticSchemaResp.getModelResps().forEach(modelResp -> { - allFields.addAll(modelResp.getFieldList()); - if (modelResp.getModelDetail().getMeasures() != null) { - modelResp.getModelDetail().getMeasures() - .forEach(measure -> allMeasures.put(measure.getBizName(), measure)); - } - }); - - Set derivedDimensions = new HashSet<>(); - Set derivedMetrics = new HashSet<>(); - Map visitedMetrics = new HashMap<>(); - - for (MetricResp metricResp : metricResps) { - if (metrics.contains(metricResp.getBizName())) { - boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(), - metricResp.getMetricDefineByMeasureParams()); - if (isDerived) { - String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, - allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp), - metricResp.getMetricDefineType(), aggOption, visitedMetrics, - derivedMetrics, derivedDimensions); - result.put(metricResp.getBizName(), expr); - log.debug("derived metric {}->{}", metricResp.getBizName(), expr); - } else { - measures.add(metricResp.getBizName()); - } - } - } - - measures.addAll(derivedMetrics); - derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension)) - .forEach(dimensions::add); - - return result; - } - - private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List dimensions) { - if (!CollectionUtils.isEmpty(dimensions)) { - Map modelMatchCnt = new HashMap<>(); - for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { - modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions() - .stream().filter(d -> dimensions.contains(d.getBizName())).count()); - } - return modelMatchCnt.entrySet().stream() - .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) - .map(m -> m.getKey()).findFirst().orElse(""); - } - return semanticSchemaResp.getModelResps().get(0).getBizName(); - } -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 597b59c9a..150bb3cda 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -140,15 +140,15 @@ public class QueryUtils { return null; } - public QueryStatement sqlParserUnion(QueryMultiStructReq queryMultiStructCmd, - List sqlParsers) { + public QueryStatement unionAll(QueryMultiStructReq queryMultiStructCmd, + List queryStatements) { QueryStatement sqlParser = new QueryStatement(); StringBuilder unionSqlBuilder = new StringBuilder(); - for (int i = 0; i < sqlParsers.size(); i++) { + for (int i = 0; i < queryStatements.size(); i++) { String selectStr = SqlGenerateUtils .getUnionSelect(queryMultiStructCmd.getQueryStructReqs().get(i)); unionSqlBuilder.append(String.format("select %s from ( %s ) sub_sql_%s", selectStr, - sqlParsers.get(i).getSql(), i)); + queryStatements.get(i).getSql(), i)); unionSqlBuilder.append(UNIONALL); } String unionSql = unionSqlBuilder.substring(0, diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index 40cd3d5db..656ccd081 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -6,8 +6,8 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; -import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; +import com.tencent.supersonic.headless.core.translator.calcite.sql.SqlBuilder; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; @@ -20,16 +20,12 @@ import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Slf4j class HeadlessParserServiceTest { - private static Map headlessSchemaMap = new HashMap<>(); - - public static SqlParserResp parser(SemanticSchema semanticSchema, + public static SqlParserResp parser(S2CalciteSchema semanticSchema, MetricQueryParam metricQueryParam, boolean isAgg) { SqlParserResp sqlParser = new SqlParserResp(); try { @@ -37,14 +33,13 @@ class HeadlessParserServiceTest { sqlParser.setErrMsg("headlessSchema not found"); return sqlParser; } - AggPlanner aggBuilder = new AggPlanner(semanticSchema); + SqlBuilder aggBuilder = new SqlBuilder(semanticSchema); QueryStatement queryStatement = new QueryStatement(); queryStatement.setMetricQueryParam(metricQueryParam); - aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); - EngineType engineType = EngineType - .fromString(semanticSchema.getSemanticModel().getDatabase().getType()); + aggBuilder.build(queryStatement, AggOption.getAggregation(!isAgg)); + EngineType engineType = + EngineType.fromString(semanticSchema.getOntology().getDatabase().getType()); sqlParser.setSql(aggBuilder.getSql(engineType)); - sqlParser.setSourceId(aggBuilder.getSourceId()); } catch (Exception e) { sqlParser.setErrMsg(e.getMessage()); log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e); @@ -122,10 +117,10 @@ class HeadlessParserServiceTest { identify.setType("primary"); identifies.add(identify); datasource.setIdentifiers(identifies); - SemanticSchema semanticSchema = SemanticSchema.newBuilder("1").build(); + S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build(); SemanticSchemaManager.update(semanticSchema, - SemanticSchemaManager.getDatasource(datasource)); + SemanticSchemaManager.getDataModel(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("page"); @@ -192,7 +187,7 @@ class HeadlessParserServiceTest { System.out.println(parser(semanticSchema, metricCommand2, true)); } - private static void addDepartment(SemanticSchema semanticSchema) { + private static void addDepartment(S2CalciteSchema semanticSchema) { DataModelYamlTpl datasource = new DataModelYamlTpl(); datasource.setName("user_department"); datasource.setSourceId(1L); @@ -238,8 +233,8 @@ class HeadlessParserServiceTest { identifies.add(identify); datasource.setIdentifiers(identifies); - semanticSchema.getDatasource().put("user_department", - SemanticSchemaManager.getDatasource(datasource)); + semanticSchema.getDataModels().put("user_department", + SemanticSchemaManager.getDataModel(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("department"); @@ -248,7 +243,7 @@ class HeadlessParserServiceTest { List dimensionYamlTpls = new ArrayList<>(); dimensionYamlTpls.add(dimension1); - semanticSchema.getDimension().put("user_department", + semanticSchema.getDimensions().put("user_department", SemanticSchemaManager.getDimensions(dimensionYamlTpls)); } } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SingerDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SingerDemo.java index 3659659b7..1b40be2d5 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SingerDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SingerDemo.java @@ -167,6 +167,8 @@ public class S2SingerDemo extends S2BaseDemo { Maps.newHashMap(ChatAppManager.getAllApps(AppModule.CHAT)); chatAppConfig.values().forEach(app -> app.setChatModelId(demoChatModel.getId())); agent.setChatAppConfig(chatAppConfig); + agent.setAdmins(Lists.newArrayList("alice")); + agent.setViewers(Lists.newArrayList("tom", "jack")); agentService.createAgent(agent, defaultUser); } } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SmallTalkDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SmallTalkDemo.java index 943ed9c3f..7964d0bf8 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SmallTalkDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SmallTalkDemo.java @@ -40,6 +40,8 @@ public class S2SmallTalkDemo extends S2BaseDemo { chatAppConfig.get(PlainTextExecutor.APP_KEY).setEnable(true); chatAppConfig.get(OnePassSCSqlGenStrategy.APP_KEY).setEnable(false); agent.setChatAppConfig(chatAppConfig); + agent.setAdmins(Lists.newArrayList("jack")); + agent.setViewers(Lists.newArrayList("alice", "tom")); agentService.createAgent(agent, defaultUser); } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java index 29748cb4c..7f7cb56b4 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java @@ -134,7 +134,7 @@ public class S2VisitsDemo extends S2BaseDemo { private void addSampleChats(Integer agentId) { Long chatId = chatManageService.addChat(defaultUser, "样例对话1", agentId); - submitText(chatId.intValue(), agentId, "超音数 访问次数"); + submitText(chatId.intValue(), agentId, "访问过超音数的部门有哪些"); submitText(chatId.intValue(), agentId, "按部门统计近7天访问次数"); submitText(chatId.intValue(), agentId, "alice 停留时长"); } @@ -162,6 +162,8 @@ public class S2VisitsDemo extends S2BaseDemo { Maps.newHashMap(ChatAppManager.getAllApps(AppModule.CHAT)); chatAppConfig.values().forEach(app -> app.setChatModelId(demoChatModel.getId())); agent.setChatAppConfig(chatAppConfig); + agent.setAdmins(Lists.newArrayList("tom")); + agent.setViewers(Lists.newArrayList("alice", "jack")); Agent agentCreated = agentService.createAgent(agent, defaultUser); return agentCreated.getId(); } @@ -444,7 +446,7 @@ public class S2VisitsDemo extends S2BaseDemo { termReq1.setDescription("用户为tom和lucy"); termReq1.setAlias(Lists.newArrayList("VIP用户")); termReq1.setDomainId(s2Domain.getId()); - termService.saveOrUpdate(termReq, defaultUser); + termService.saveOrUpdate(termReq1, defaultUser); } private void addAuthGroup_1(ModelResp stayTimeModel) { diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index 0cefd0fb6..17c43e127 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -393,4 +393,8 @@ ALTER TABLE s2_agent DROP COLUMN `multi_turn_config`; ALTER TABLE s2_agent DROP COLUMN `enable_memory_review`; --20241012 -alter table s2_agent add column `enable_feedback` tinyint DEFAULT 1; \ No newline at end of file +alter table s2_agent add column `enable_feedback` tinyint DEFAULT 1; + +--20241116 +alter table s2_agent add column `admin` varchar(1000); +alter table s2_agent add column `viewer` varchar(1000); \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 71985491c..4da6f27c2 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -398,6 +398,8 @@ CREATE TABLE IF NOT EXISTS s2_agent updated_at TIMESTAMP null, enable_search int null, enable_feedback int null, + admin varchar(1000), + viewer varchar(1000), PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_agent IS 'agent information table'; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java index d813ef0aa..e93e5bcb6 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java @@ -50,8 +50,7 @@ public class SchemaAuthTest extends BaseTest { @Test public void test_getVisibleModelList_alice() { User user = DataUtils.getUserAlice(); - List modelResps = - modelService.getModelListWithAuth(user, null, AuthType.VISIBLE); + List modelResps = modelService.getModelListWithAuth(user, null, AuthType.VIEWER); List expectedModelBizNames = Lists.newArrayList("user_department", "singer"); Assertions.assertEquals(expectedModelBizNames, modelResps.stream().map(ModelResp::getBizName).collect(Collectors.toList())); diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 7439debac..8584c6239 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -398,6 +398,8 @@ CREATE TABLE IF NOT EXISTS s2_agent updated_at TIMESTAMP null, enable_search int null, enable_feedback int null, + admin varchar(1000), + viewer varchar(1000), PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_agent IS 'agent information table'; diff --git a/webapp/packages/chat-sdk/src/utils/utils.ts b/webapp/packages/chat-sdk/src/utils/utils.ts index de8494240..b514a0105 100644 --- a/webapp/packages/chat-sdk/src/utils/utils.ts +++ b/webapp/packages/chat-sdk/src/utils/utils.ts @@ -3,6 +3,9 @@ import { NumericUnit } from '../common/constants'; import { isString } from 'lodash'; export function formatByDecimalPlaces(value: number | string, decimalPlaces: number) { + if (value === null || value === undefined || value === '') { + return 0; + } if (isNaN(+value) || decimalPlaces < 0 || decimalPlaces > 100) { return value; } @@ -17,6 +20,9 @@ export function formatByDecimalPlaces(value: number | string, decimalPlaces: num } export function formatByThousandSeperator(value: number | string) { + if (value === null || value === undefined || value === '') { + return 0; + } if (isNaN(+value)) { return value; } diff --git a/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.less b/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.less index 4067901a5..ec93cc852 100644 --- a/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.less +++ b/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.less @@ -7,3 +7,12 @@ .userText { margin-left: 10px; } + +.selectPerson { + :global { + .ant-select-selection-item { + color: rgba(0, 0, 0, 0.88)!important; + background-color: rgba(0, 0, 0, 0.06)!important; + } + } +} \ No newline at end of file diff --git a/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.tsx b/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.tsx index 7c50c8c91..0a1f5d20b 100644 --- a/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.tsx +++ b/webapp/packages/supersonic-fe/src/components/SelectTMEPerson/index.tsx @@ -37,6 +37,7 @@ const SelectTMEPerson: FC = ({ placeholder, value, isMultiple = true, onC mode={isMultiple ? 'multiple' : undefined} allowClear showSearch + className={styles.selectPerson} onChange={onChange} > {userList.map((item) => { diff --git a/webapp/packages/supersonic-fe/src/pages/Agent/AgentForm.tsx b/webapp/packages/supersonic-fe/src/pages/Agent/AgentForm.tsx index 6c4c00e60..4139478c5 100644 --- a/webapp/packages/supersonic-fe/src/pages/Agent/AgentForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/Agent/AgentForm.tsx @@ -8,6 +8,8 @@ import { uuid, jsonParse } from '@/utils/utils'; import ToolsSection from './ToolsSection'; import globalStyles from '@/global.less'; import { QuestionCircleOutlined } from '@ant-design/icons'; +import SelectTMEPerson from '@/components/SelectTMEPerson'; +import FormItemTitle from '@/components/FormHelper/FormItemTitle'; import { getLlmModelTypeList, getLlmModelAppList, getLlmList } from '../../services/system'; import MemorySection from './MemorySection'; @@ -223,7 +225,16 @@ const AgentForm: React.FC = ({ editAgent, onSaveAgent, onCreateToolBtnCli > - + + + + + +
{examples.map((example) => { diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx index 50cfaa67b..249f8dccb 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx @@ -207,26 +207,26 @@ const ModelFieldForm: React.FC = ({ // width: 200, render: (_: any, record: FieldItem) => { const { type } = record; - if (type === EnumDataSourceType.PRIMARY) { - return ( - - { + // onTagObjectChange?.(value); + // }} + // options={tagObjectList.map((item: ISemantic.ITagObjectItem) => { + // return { + // label: item.name, + // value: item.id, + // }; + // })} + // /> + // + // ); + // } if (type === EnumDataSourceType.MEASURES) { return (