(improvement)(headless) fixed parser error (#1161) (#1166)

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.core.parser.calcite;
 
 
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
-import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl;
 import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
+import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl;
 import com.tencent.supersonic.headless.core.parser.calcite.schema.ViewExpanderImpl;
 import com.tencent.supersonic.headless.core.utils.SqlDialectFactory;
 import java.util.ArrayList;
@@ -27,6 +27,8 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeSystem;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.advise.SqlAdvisor;
+import org.apache.calcite.sql.advise.SqlAdvisorValidator;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.parser.SqlParser;
 import org.apache.calcite.sql.parser.impl.SqlParserImpl;
@@ -34,6 +36,7 @@ import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
 import org.apache.calcite.sql.validate.SqlValidator;
 import org.apache.calcite.sql.validate.SqlValidatorScope;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.sql.validate.SqlValidatorWithHints;
 import org.apache.calcite.sql2rel.SqlToRelConverter;
 import org.apache.calcite.tools.FrameworkConfig;
 import org.apache.calcite.tools.Frameworks;
@@ -96,6 +99,13 @@ public class Configuration {
                 Configuration.getValidatorConfig(engineType));
     }
 
+    public static SqlValidatorWithHints getSqlValidatorWithHints(CalciteSchema rootSchema, EngineType engineTyp) {
+        return new SqlAdvisorValidator(SqlStdOperatorTable.instance(),
+                new CalciteCatalogReader(rootSchema,
+                        Collections.singletonList(rootSchema.getName()), typeFactory, config),
+                typeFactory, SqlValidator.Config.DEFAULT);
+    }
+
     public static SqlToRelConverter.Config getConverterConfig() {
         HintStrategyTable strategies = HintStrategyTable.builder().build();
         return SqlToRelConverter.config()
@@ -119,4 +129,8 @@ public class Configuration {
                 getConverterConfig());
     }
 
+    public static SqlAdvisor getSqlAdvisor(SqlValidatorWithHints validator, EngineType engineType) {
+        return new SqlAdvisor(validator, getParserConfig(engineType));
+    }
+
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java

@@ -80,12 +80,14 @@ public class SchemaBuilder {
                 .withRowCount(1)
                 .build();
         if (Objects.nonNull(db) && !db.isEmpty()) {
-            SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db);
+            SchemaPlus dbPs = dataSetSchema.plus();
-            if (Objects.isNull(schemaPlus)) {
+            for (String d : db.split("\\.")) {
-                dataSetSchema.plus().add(db, new AbstractSchema());
+                if (dbPs.getSubSchema(d) == null) {
-                schemaPlus = dataSetSchema.plus().getSubSchema(db);
+                    dbPs.add(d, new AbstractSchema());
                 }
-            schemaPlus.add(tb, srcTable);
+                dbPs = dbPs.getSubSchema(d);
+            }
+            dbPs.add(tb, srcTable);
         } else {
             dataSetSchema.add(tb, srcTable);
         }

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
 
 import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
@@ -55,29 +56,20 @@ public class DataSourceNode extends SemanticNode {
             throw new Exception("DatasourceNode build error [tableSqlNode not found]");
         }
         SqlNode source = getTable(sqlTable, scope, EngineType.fromString(datasource.getType()));
-        addSchema(scope, datasource, source);
+        addSchema(scope, datasource, sqlTable);
         return buildAs(datasource.getName(), source);
     }
 
-    private static void addSchema(SqlValidatorScope scope, DataSource datasource, SqlNode table) throws Exception {
+    private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) throws Exception {
-        Map<String, Object> parseInfo = SemanticNode.getDbTable(table);
+        Map<String, Set<String>> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table);
-        if (!parseInfo.isEmpty() && parseInfo.containsKey(Constants.SQL_PARSER_TABLE)) {
+        for (Map.Entry<String, Set<String>> entry : sqlTable.entrySet()) {
-            Map<String, Set<String>> dbTbs = (Map<String, Set<String>>) parseInfo.get(Constants.SQL_PARSER_TABLE);
+            String tb = entry.getKey();
-            Map<String, Set<String>> fields = (Map<String, Set<String>>) parseInfo.get(Constants.SQL_PARSER_FIELD);
+            String db = "";
-            for (Map.Entry<String, Set<String>> entry : dbTbs.entrySet()) {
+            if (entry.getKey().indexOf(".") > 0) {
-                for (String dbTb : entry.getValue()) {
+                db = entry.getKey().substring(0, entry.getKey().indexOf("."));
-                    String[] dbTable = dbTb.split("\\.");
+                tb = entry.getKey().substring(entry.getKey().indexOf(".") + 1);
-                    if (Objects.nonNull(dbTable) && dbTable.length > 0) {
-                        String tb = dbTable.length > 1 ? dbTable[1] : dbTable[0];
-                        String db = dbTable.length > 1 ? dbTable[0] : "";
-                        addSchemaTable(scope, datasource, db, tb,
-                                fields.containsKey(entry.getKey()) ? fields.get(entry.getKey())
-                                        : dbTbs.size() == 1 && fields.size() == 1 && fields.containsKey("")
-                                                ? fields.get("")
-                                                : new HashSet<>());
-                    }
-                }
             }
+            addSchemaTable(scope, datasource, db, tb, entry.getValue());
         }
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java

@@ -46,6 +46,7 @@ import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.pretty.SqlPrettyWriter;
 import org.apache.calcite.sql.validate.SqlValidator;
 import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.sql.validate.SqlValidatorWithHints;
 import org.apache.calcite.sql2rel.SqlToRelConverter;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -75,6 +76,12 @@ public abstract class SemanticNode {
     }
 
     public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType) throws Exception {
+        SqlValidatorWithHints sqlValidatorWithHints = Configuration.getSqlValidatorWithHints(
+                scope.getValidator().getCatalogReader().getRootSchema(), engineType);
+        if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords()
+                .contains(expression.toUpperCase())) {
+            expression = String.format("`%s`", expression);
+        }
         SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig(engineType));
         SqlNode sqlNode = sqlParser.parseExpression();
         scope.validateExpr(sqlNode);

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java

@@ -156,7 +156,7 @@ public class QueryReqConverter {
         if (databaseReq.isInnerLayerNative()) {
             return AggOption.NATIVE;
         }
-        if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)) {
+        if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) {
             return AggOption.NATIVE;
         }
         long defaultAggNullCnt = metricSchemas.stream()