mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 11:07:06 +00:00
Squashed commit of the following: (#598)
This commit is contained in:
jipeli
@@ -1,5 +1,10 @@
|
||||
package com.tencent.supersonic.headless.api.enums;
|
||||
|
||||
/**
|
||||
* Aggregation type of metric when query metric without aggregation method
|
||||
* NATIVE: will not use Aggregation
|
||||
* DEFAULT: will use the aggregation method define in the model
|
||||
*/
|
||||
public enum AggOption {
|
||||
NATIVE,
|
||||
AGGREGATION,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package com.tencent.supersonic.headless.api.enums;
|
||||
|
||||
|
||||
/**
|
||||
* model datasource define type:
|
||||
* sql_query : view sql begin as select
|
||||
* table_query: dbName.tableName
|
||||
*/
|
||||
public enum DatasourceQuery {
|
||||
|
||||
SQL_QUERY("sql_query"),
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
package com.tencent.supersonic.headless.api.enums;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* model source table type:
|
||||
* FULL: table without time partition
|
||||
* PARTITION: table with partition field
|
||||
* ZIPPER: table with slowly changing dimension
|
||||
*/
|
||||
public enum ModelSourceType {
|
||||
FULL,
|
||||
PARTITION,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package com.tencent.supersonic.headless.core.adaptor.db;
|
||||
|
||||
|
||||
/**
|
||||
* Adapters for different query engines to obtain table, field, and time formatting methods
|
||||
*/
|
||||
public abstract class DbAdaptor {
|
||||
|
||||
public abstract String getDateFormat(String dateType, String dateFormat, String column);
|
||||
|
||||
@@ -3,6 +3,9 @@ package com.tencent.supersonic.headless.core.executor;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
/**
|
||||
* Query data or execute sql from a query engine
|
||||
*/
|
||||
public interface QueryExecutor {
|
||||
|
||||
boolean accept(QueryStatement queryStatement);
|
||||
|
||||
@@ -13,7 +13,9 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
/**
|
||||
* manager to handle the dimension
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DimensionYamlManager {
|
||||
|
||||
@@ -15,7 +15,9 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
/**
|
||||
* manager to handle the metric
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MetricYamlManager {
|
||||
|
||||
@@ -24,7 +24,9 @@ import org.springframework.util.CollectionUtils;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
/**
|
||||
* manager to handle the model
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ModelYamlManager {
|
||||
|
||||
@@ -10,6 +10,9 @@ import org.springframework.util.CollectionUtils;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Remove the default metric added by the system when the query only has dimensions
|
||||
*/
|
||||
@Slf4j
|
||||
@Component("DetailQuery")
|
||||
public class DetailQuery implements QueryOptimizer {
|
||||
|
||||
@@ -3,6 +3,9 @@ package com.tencent.supersonic.headless.core.optimizer;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
/**
|
||||
* the interface that rewrites QueryStatement with some optimization rules
|
||||
*/
|
||||
public interface QueryOptimizer {
|
||||
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ package com.tencent.supersonic.headless.core.parser;
|
||||
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
/**
|
||||
* to supplement,translate the request Body
|
||||
*/
|
||||
public interface HeadlessConverter {
|
||||
|
||||
boolean accept(QueryStatement queryStatement);
|
||||
|
||||
@@ -19,6 +19,9 @@ import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* logical parse from ParseSqlReq or MetricReq
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
@Primary
|
||||
|
||||
@@ -10,6 +10,9 @@ import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* the calcite parse implements
|
||||
*/
|
||||
@Component("CalciteSqlParser")
|
||||
public class CalciteSqlParser implements SqlParser {
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ import org.apache.calcite.sql2rel.SqlToRelConverter;
|
||||
import org.apache.calcite.tools.FrameworkConfig;
|
||||
import org.apache.calcite.tools.Frameworks;
|
||||
|
||||
/**
|
||||
* global configuration of the calcite
|
||||
*/
|
||||
public class Configuration {
|
||||
|
||||
public static Properties configProperties = new Properties();
|
||||
@@ -94,7 +97,8 @@ public class Configuration {
|
||||
return SqlToRelConverter.config()
|
||||
.withHintStrategyTable(strategies)
|
||||
.withTrimUnusedFields(true)
|
||||
.withExpand(true);
|
||||
.withExpand(true)
|
||||
.addRelBuilderConfigTransform(c -> c.withSimplify(false));
|
||||
}
|
||||
|
||||
public static SqlToRelConverter getSqlToRelConverter(SqlValidatorScope scope, SqlValidator sqlValidator,
|
||||
|
||||
@@ -36,6 +36,9 @@ import java.util.ListIterator;
|
||||
import java.util.Objects;
|
||||
import java.util.Stack;
|
||||
|
||||
/**
|
||||
* parsing from query dimensions and metrics
|
||||
*/
|
||||
@Slf4j
|
||||
public class AggPlanner implements Planner {
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
|
||||
/**
|
||||
* parse and generate SQL and other execute information
|
||||
*/
|
||||
public interface Planner {
|
||||
|
||||
public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception;
|
||||
|
||||
@@ -22,6 +22,9 @@ import org.apache.calcite.schema.TranslatableTable;
|
||||
import org.apache.calcite.schema.impl.AbstractTable;
|
||||
import org.apache.calcite.sql.type.SqlTypeName;
|
||||
|
||||
/**
|
||||
* customize the AbstractTable
|
||||
*/
|
||||
public class DataSourceTable extends AbstractTable implements ScannableTable, TranslatableTable {
|
||||
|
||||
private final String tableName;
|
||||
|
||||
@@ -16,6 +16,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class HeadlessSchema extends AbstractSchema {
|
||||
|
||||
private final String rootPath;
|
||||
|
||||
@@ -4,6 +4,9 @@ import org.apache.calcite.sql.fun.SqlLibrary;
|
||||
import org.apache.calcite.sql.validate.SqlConformance;
|
||||
import org.apache.calcite.sql.validate.SqlConformanceEnum;
|
||||
|
||||
/**
|
||||
* customize the SqlConformance
|
||||
*/
|
||||
public class HeadlessSqlConformance implements SqlConformance {
|
||||
|
||||
@Override
|
||||
|
||||
@@ -4,6 +4,9 @@ import java.nio.charset.Charset;
|
||||
import org.apache.calcite.rel.type.RelDataTypeSystem;
|
||||
import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
|
||||
|
||||
/**
|
||||
* customize the SqlTypeFactoryImpl
|
||||
*/
|
||||
public class HeadlessSqlTypeFactoryImpl extends SqlTypeFactoryImpl {
|
||||
|
||||
public HeadlessSqlTypeFactoryImpl(RelDataTypeSystem typeSystem) {
|
||||
|
||||
@@ -10,6 +10,9 @@ import org.apache.calcite.sql.SqlWriter;
|
||||
import org.apache.calcite.sql.validate.SqlConformance;
|
||||
import org.checkerframework.checker.nullness.qual.Nullable;
|
||||
|
||||
/**
|
||||
* customize the SqlDialect
|
||||
*/
|
||||
public class SemanticSqlDialect extends SqlDialect {
|
||||
|
||||
public static final Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT
|
||||
|
||||
@@ -5,6 +5,9 @@ import org.apache.calcite.plan.RelOptTable;
|
||||
import org.apache.calcite.rel.RelRoot;
|
||||
import org.apache.calcite.rel.type.RelDataType;
|
||||
|
||||
/**
|
||||
* customize the ViewExpander
|
||||
*/
|
||||
public class ViewExpanderImpl implements RelOptTable.ViewExpander {
|
||||
public ViewExpanderImpl() {
|
||||
}
|
||||
|
||||
@@ -20,6 +20,9 @@ import lombok.Data;
|
||||
import org.apache.calcite.sql.SqlNode;
|
||||
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
||||
|
||||
/**
|
||||
* process TableView
|
||||
*/
|
||||
@Data
|
||||
public abstract class Renderer {
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ import org.apache.calcite.sql.SqlOperatorTable;
|
||||
import org.apache.calcite.sql.validate.SqlValidatorCatalogReader;
|
||||
import org.apache.calcite.sql.validate.SqlValidatorImpl;
|
||||
|
||||
/**
|
||||
* customize the SqlValidatorImpl
|
||||
*/
|
||||
public class S2SQLSqlValidatorImpl extends SqlValidatorImpl {
|
||||
|
||||
public S2SQLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
|
||||
|
||||
@@ -12,6 +12,9 @@ import org.apache.calcite.sql.SqlNodeList;
|
||||
import org.apache.calcite.sql.SqlSelect;
|
||||
import org.apache.calcite.sql.parser.SqlParserPos;
|
||||
|
||||
/**
|
||||
* basic query project
|
||||
*/
|
||||
@Data
|
||||
public class TableView {
|
||||
|
||||
|
||||
@@ -42,6 +42,9 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
|
||||
import org.apache.calcite.sql2rel.SqlToRelConverter;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* model item node
|
||||
*/
|
||||
public abstract class SemanticNode {
|
||||
|
||||
public static Set<SqlKind> AGGREGATION_KIND = new HashSet<>();
|
||||
|
||||
@@ -11,6 +11,9 @@ import org.apache.calcite.sql.SqlNodeList;
|
||||
import org.apache.calcite.sql.SqlOperator;
|
||||
import org.apache.calcite.sql.SqlWriter;
|
||||
|
||||
/**
|
||||
* extend node to handle lateral explode view
|
||||
*/
|
||||
public class LateralViewExplodeNode extends ExtendNode {
|
||||
|
||||
public final String sqlNameView = "view";
|
||||
|
||||
@@ -26,6 +26,10 @@ import org.apache.calcite.util.ImmutableBitSet;
|
||||
import org.apache.calcite.util.Pair;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
|
||||
/**
|
||||
* push down the time filter into group using the RuntimeOptions defined minMaxTime
|
||||
*
|
||||
*/
|
||||
public class FilterToGroupScanRule extends RelRule<Config>
|
||||
implements TransformationRule {
|
||||
|
||||
|
||||
@@ -23,6 +23,9 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* process query specified filtering information
|
||||
*/
|
||||
public class FilterRender extends Renderer {
|
||||
|
||||
@Override
|
||||
|
||||
@@ -41,6 +41,9 @@ import java.util.Queue;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* process the join conditions when the source number is greater than 1
|
||||
*/
|
||||
@Slf4j
|
||||
public class JoinRender extends Renderer {
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ import org.apache.calcite.sql.parser.SqlParserPos;
|
||||
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
/**
|
||||
* process the query result items from query request
|
||||
*/
|
||||
public class OutputRender extends Renderer {
|
||||
|
||||
@Override
|
||||
|
||||
@@ -32,6 +32,9 @@ import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* process the table view from the defined data model schema
|
||||
*/
|
||||
@Slf4j
|
||||
public class SourceRender extends Renderer {
|
||||
|
||||
|
||||
@@ -24,7 +24,9 @@ import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
/**
|
||||
* supplement the QueryStatement when query with custom aggregation method
|
||||
*/
|
||||
@Component("CalculateAggConverter")
|
||||
@Slf4j
|
||||
public class CalculateAggConverter implements HeadlessConverter {
|
||||
|
||||
@@ -17,6 +17,9 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* HeadlessConverter default implement
|
||||
*/
|
||||
@Component("ParserDefaultConverter")
|
||||
@Slf4j
|
||||
public class ParserDefaultConverter implements HeadlessConverter {
|
||||
|
||||
@@ -16,6 +16,9 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* HeadlessConverter QueryOptimizer QueryExecutor object factory
|
||||
*/
|
||||
public class ComponentFactory {
|
||||
|
||||
private static List<HeadlessConverter> headlessConverters = new ArrayList<>();
|
||||
|
||||
@@ -11,6 +11,9 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* transform query results to return the users
|
||||
*/
|
||||
public class DataTransformUtils {
|
||||
|
||||
public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData, String metric,
|
||||
|
||||
@@ -27,6 +27,9 @@ import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
|
||||
|
||||
/**
|
||||
* tools functions about jdbc
|
||||
*/
|
||||
@Slf4j
|
||||
public class JdbcDataSourceUtils {
|
||||
|
||||
|
||||
@@ -35,6 +35,9 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
|
||||
|
||||
/**
|
||||
* tools functions to analyze queryStructReq
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class SqlGenerateUtils {
|
||||
|
||||
@@ -29,7 +29,9 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
/**
|
||||
* tools functions about sql query
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class SqlUtils {
|
||||
|
||||
@@ -9,6 +9,9 @@ import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* generate system time dimension tools
|
||||
*/
|
||||
@Slf4j
|
||||
public class SysTimeDimensionBuilder {
|
||||
|
||||
|
||||
Reference in New Issue
Block a user