topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

support foreign identify (#111)

Browse Source 
* [improvement][semantic] add get metric agg function

* [improvement][semantic] support foreign identify
This commit is contained in:
jipeli
2023-09-20 22:20:00 +08:00
committed by GitHub
parent 071ef8432e
commit 6dfc728b5b
6 changed files with 141 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -221,7 +221,7 @@ public class ModelServiceImpl implements ModelService {
@Override
public Map<Long, String> getModelFullPathMap() {
return getModelList().stream().collect(Collectors.toMap(ModelResp::getId,
return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId,
ModelResp::getFullPath, (k1, k2) -> k1));
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java
View File

@@ -10,6 +10,10 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class Identify {
public enum Type {
PRIMARY, FOREIGN
}
private String name;
// primary or foreign

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -25,6 +26,8 @@ public class TableView {
private String alias;
private List<String> primary;
private DataSource dataSource;
public SqlNode build() {
measure.addAll(dimension);
SqlNodeList dimensionNodeList = null;

29
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java
View File

@@ -1,6 +1,11 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;
@@ -9,4 +14,28 @@ public class IdentifyNode extends SemanticNode {
public static SqlNode build(Identify identify, SqlValidatorScope scope) throws Exception {
return parse(identify.getName(), scope);
}
public static Set<String> getIdentifyNames(List<Identify> identifies, Identify.Type type) {
return identifies.stream().filter(i -> type.name().equalsIgnoreCase(i.getType())).map(i -> i.getName())
.collect(Collectors.toSet());
}
public static boolean isForeign(String name, List<Identify> identifies) {
Optional<Identify> identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst();
if (identify.isPresent()) {
return Type.FOREIGN.name().equalsIgnoreCase(identify.get().getType());
}
return false;
}
public static boolean isPrimary(String name, List<Identify> identifies) {
Optional<Identify> identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst();
if (identify.isPresent()) {
return Type.PRIMARY.name().equalsIgnoreCase(identify.get().getType());
}
return false;
}
}

102
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
@@ -12,15 +13,20 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@@ -33,6 +39,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class JoinRender extends Renderer {
@@ -41,6 +48,7 @@ public class JoinRender extends Renderer {
public void render(MetricReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
dataSources = getOrderSource(dataSources);
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
if (queryWhere != null && !queryWhere.isEmpty()) {
@@ -95,6 +103,7 @@ public class JoinRender extends Renderer {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
tableView.setAlias(alias);
tableView.setPrimary(primary);
tableView.setDataSource(dataSource);
if (left == null) {
leftTable = tableView;
left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope));
@@ -246,7 +255,7 @@ public class JoinRender extends Renderer {
private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema,
SqlValidatorScope scope) throws Exception {
log.info(left.getClass().toString());
Set<String> selectLeft = SemanticNode.getSelect(left.getTable());
Set<String> selectRight = SemanticNode.getSelect(right.getTable());
selectLeft.retainAll(selectRight);
@@ -255,6 +264,16 @@ public class JoinRender extends Renderer {
if (!SourceRender.isDimension(on, dataSource, schema)) {
continue;
}
if (IdentifyNode.isForeign(on, left.getDataSource().getIdentifiers())) {
if (!IdentifyNode.isPrimary(on, right.getDataSource().getIdentifiers())) {
continue;
}
}
if (IdentifyNode.isForeign(on, right.getDataSource().getIdentifiers())) {
if (!IdentifyNode.isPrimary(on, left.getDataSource().getIdentifiers())) {
continue;
}
}
List<SqlNode> ons = new ArrayList<>();
ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope));
ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope));
@@ -276,4 +295,85 @@ public class JoinRender extends Renderer {
}
return condition;
}
private List<DataSource> getOrderSource(List<DataSource> dataSources) throws Exception {
if (CollectionUtils.isEmpty(dataSources) || dataSources.size() <= 2) {
return dataSources;
}
Map<String, Set<String>> next = new HashMap<>();
Map<String, Boolean> visited = new HashMap<>();
Map<String, List<Identify>> dataSourceIdentifies = new HashMap<>();
dataSources.stream().forEach(d -> {
next.put(d.getName(), new HashSet<>());
visited.put(d.getName(), false);
dataSourceIdentifies.put(d.getName(), d.getIdentifiers());
});
int cnt = dataSources.size();
List<Map.Entry<String, List<Identify>>> dataSourceIdentifyList = dataSourceIdentifies.entrySet().stream()
.collect(
Collectors.toList());
for (int i = 0; i < cnt; i++) {
for (int j = i + 1; j < cnt; j++) {
Set<String> primaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(),
Type.PRIMARY);
Set<String> foreign = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(),
Type.FOREIGN);
Set<String> nextPrimaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(j).getValue(),
Type.PRIMARY);
Set<String> nextForeign = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(j).getValue(),
Type.FOREIGN);
Set<String> nextAll = new HashSet<>();
nextAll.addAll(nextPrimaries);
nextAll.addAll(nextForeign);
primaries.retainAll(nextPrimaries);
foreign.retainAll(nextPrimaries);
if (primaries.size() > 0 || foreign.size() > 0) {
next.get(dataSourceIdentifyList.get(i).getKey()).add(dataSourceIdentifyList.get(j).getKey());
next.get(dataSourceIdentifyList.get(j).getKey()).add(dataSourceIdentifyList.get(i).getKey());
}
}
}
Queue<String> paths = new ArrayDeque<>();
for (String id : visited.keySet()) {
if (!visited.get(id)) {
joinOrder(cnt, id, next, paths, visited);
if (paths.size() >= cnt) {
break;
}
}
}
if (paths.size() < cnt) {
throw new Exception("datasource cant join,pls check identify :" + dataSources.stream()
.map(d -> d.getName()).collect(
Collectors.joining(",")));
}
List<String> orderList = new ArrayList<>(paths);
Collections.sort(dataSources, new Comparator<DataSource>() {
@Override
public int compare(DataSource o1, DataSource o2) {
return orderList.indexOf(o1.getName()) - orderList.indexOf(o2.getName());
}
});
return dataSources;
}
private static void joinOrder(int cnt, String id, Map<String, Set<String>> next, Queue<String> orders,
Map<String, Boolean> visited) {
visited.put(id, true);
orders.add(id);
if (orders.size() >= cnt) {
return;
}
for (String nextId : next.get(id)) {
if (!visited.get(nextId)) {
joinOrder(cnt, nextId, next, orders, visited);
if (orders.size() >= cnt) {
return;
}
}
}
orders.poll();
visited.put(id, false);
}
}

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java
View File

@@ -5,10 +5,10 @@ import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Criterion;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -25,6 +25,7 @@ import org.springframework.util.CollectionUtils;
public class SqlFilterUtils {
private static String pattern = "^'.*?'$";
private static String numericPattern = "^[0-9]+$";
public List<String> getFiltersCol(List<Filter> filters) {
List<String> filterCols = new ArrayList<>();
@@ -219,7 +220,7 @@ public class SqlFilterUtils {
}
private String valueApostropheLogic(String value) {
if (Pattern.matches(pattern, value)) {
if (Pattern.matches(pattern, value) || Pattern.matches(numericPattern, value)) {
return value;
}
return Constants.APOSTROPHE + value + Constants.APOSTROPHE;