diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 5c5cf2bca..aedaf2250 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -221,7 +221,7 @@ public class ModelServiceImpl implements ModelService { @Override public Map getModelFullPathMap() { - return getModelList().stream().collect(Collectors.toMap(ModelResp::getId, + return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId, ModelResp::getFullPath, (k1, k2) -> k1)); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java index 8dc4173d5..61458caca 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java @@ -10,6 +10,10 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class Identify { + public enum Type { + PRIMARY, FOREIGN + } + private String name; // primary or foreign diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java index db6893612..0659b74d3 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql; +import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -25,6 +26,8 @@ public class TableView { private String alias; private List primary; + private DataSource dataSource; + public SqlNode build() { measure.addAll(dimension); SqlNodeList dimensionNodeList = null; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java index 796edb35a..1203ad739 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java @@ -1,6 +1,11 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; @@ -9,4 +14,28 @@ public class IdentifyNode extends SemanticNode { public static SqlNode build(Identify identify, SqlValidatorScope scope) throws Exception { return parse(identify.getName(), scope); } + + public static Set getIdentifyNames(List identifies, Identify.Type type) { + return identifies.stream().filter(i -> type.name().equalsIgnoreCase(i.getType())).map(i -> i.getName()) + .collect(Collectors.toSet()); + + } + + public static boolean isForeign(String name, List identifies) { + Optional identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)) + .findFirst(); + if (identify.isPresent()) { + return Type.FOREIGN.name().equalsIgnoreCase(identify.get().getType()); + } + return false; + } + + public static boolean isPrimary(String name, List identifies) { + Optional identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)) + .findFirst(); + if (identify.isPresent()) { + return Type.PRIMARY.name().equalsIgnoreCase(identify.get().getType()); + } + return false; + } } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java index a83f7d70e..c61dca9a0 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; @@ -12,15 +13,20 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -33,6 +39,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.springframework.util.CollectionUtils; @Slf4j public class JoinRender extends Renderer { @@ -41,6 +48,7 @@ public class JoinRender extends Renderer { public void render(MetricReq metricCommand, List dataSources, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); + dataSources = getOrderSource(dataSources); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); if (queryWhere != null && !queryWhere.isEmpty()) { @@ -95,6 +103,7 @@ public class JoinRender extends Renderer { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); tableView.setAlias(alias); tableView.setPrimary(primary); + tableView.setDataSource(dataSource); if (left == null) { leftTable = tableView; left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)); @@ -246,7 +255,7 @@ public class JoinRender extends Renderer { private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, SqlValidatorScope scope) throws Exception { - log.info(left.getClass().toString()); + Set selectLeft = SemanticNode.getSelect(left.getTable()); Set selectRight = SemanticNode.getSelect(right.getTable()); selectLeft.retainAll(selectRight); @@ -255,6 +264,16 @@ public class JoinRender extends Renderer { if (!SourceRender.isDimension(on, dataSource, schema)) { continue; } + if (IdentifyNode.isForeign(on, left.getDataSource().getIdentifiers())) { + if (!IdentifyNode.isPrimary(on, right.getDataSource().getIdentifiers())) { + continue; + } + } + if (IdentifyNode.isForeign(on, right.getDataSource().getIdentifiers())) { + if (!IdentifyNode.isPrimary(on, left.getDataSource().getIdentifiers())) { + continue; + } + } List ons = new ArrayList<>(); ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope)); ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope)); @@ -276,4 +295,85 @@ public class JoinRender extends Renderer { } return condition; } + + private List getOrderSource(List dataSources) throws Exception { + if (CollectionUtils.isEmpty(dataSources) || dataSources.size() <= 2) { + return dataSources; + } + Map> next = new HashMap<>(); + Map visited = new HashMap<>(); + Map> dataSourceIdentifies = new HashMap<>(); + dataSources.stream().forEach(d -> { + next.put(d.getName(), new HashSet<>()); + visited.put(d.getName(), false); + dataSourceIdentifies.put(d.getName(), d.getIdentifiers()); + }); + int cnt = dataSources.size(); + List>> dataSourceIdentifyList = dataSourceIdentifies.entrySet().stream() + .collect( + Collectors.toList()); + for (int i = 0; i < cnt; i++) { + for (int j = i + 1; j < cnt; j++) { + Set primaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(), + Type.PRIMARY); + Set foreign = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(), + Type.FOREIGN); + Set nextPrimaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(j).getValue(), + Type.PRIMARY); + Set nextForeign = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(j).getValue(), + Type.FOREIGN); + Set nextAll = new HashSet<>(); + nextAll.addAll(nextPrimaries); + nextAll.addAll(nextForeign); + primaries.retainAll(nextPrimaries); + foreign.retainAll(nextPrimaries); + if (primaries.size() > 0 || foreign.size() > 0) { + next.get(dataSourceIdentifyList.get(i).getKey()).add(dataSourceIdentifyList.get(j).getKey()); + next.get(dataSourceIdentifyList.get(j).getKey()).add(dataSourceIdentifyList.get(i).getKey()); + } + + } + } + Queue paths = new ArrayDeque<>(); + for (String id : visited.keySet()) { + if (!visited.get(id)) { + joinOrder(cnt, id, next, paths, visited); + if (paths.size() >= cnt) { + break; + } + } + } + if (paths.size() < cnt) { + throw new Exception("datasource cant join,pls check identify :" + dataSources.stream() + .map(d -> d.getName()).collect( + Collectors.joining(","))); + } + List orderList = new ArrayList<>(paths); + Collections.sort(dataSources, new Comparator() { + @Override + public int compare(DataSource o1, DataSource o2) { + return orderList.indexOf(o1.getName()) - orderList.indexOf(o2.getName()); + } + }); + return dataSources; + } + + private static void joinOrder(int cnt, String id, Map> next, Queue orders, + Map visited) { + visited.put(id, true); + orders.add(id); + if (orders.size() >= cnt) { + return; + } + for (String nextId : next.get(id)) { + if (!visited.get(nextId)) { + joinOrder(cnt, nextId, next, orders, visited); + if (orders.size() >= cnt) { + return; + } + } + } + orders.poll(); + visited.put(id, false); + } } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java index 40f8caa93..afdfd88a9 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java @@ -5,10 +5,10 @@ import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START; import static com.tencent.supersonic.common.pojo.Constants.SPACE; import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR; +import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Criterion; import com.tencent.supersonic.semantic.api.query.pojo.Filter; -import com.tencent.supersonic.common.pojo.Constants; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -25,6 +25,7 @@ import org.springframework.util.CollectionUtils; public class SqlFilterUtils { private static String pattern = "^'.*?'$"; + private static String numericPattern = "^[0-9]+$"; public List getFiltersCol(List filters) { List filterCols = new ArrayList<>(); @@ -219,7 +220,7 @@ public class SqlFilterUtils { } private String valueApostropheLogic(String value) { - if (Pattern.matches(pattern, value)) { + if (Pattern.matches(pattern, value) || Pattern.matches(numericPattern, value)) { return value; } return Constants.APOSTROPHE + value + Constants.APOSTROPHE;