[improvement][chat]Opt log infos.

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ErrorMsgRewriteProcessor.java

@@ -40,7 +40,7 @@ public class ErrorMsgRewriteProcessor implements ParseResultProcessor {
         ChatAppManager.register(APP_KEY_ERROR_MESSAGE,
                 ChatApp.builder().prompt(REWRITE_ERROR_MESSAGE_INSTRUCTION).name("异常提示改写")
                         .appModule(AppModule.CHAT).description("通过大模型将异常信息改写为更友好和引导性的提示用语")
-                        .enable(false).build());
+                        .enable(true).build());
     }
 
     @Override

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java

@@ -423,8 +423,10 @@ public class SqlReplaceHelper {
             painSelect.accept(new SelectVisitorAdapter() {
                 @Override
                 public void visit(PlainSelect plainSelect) {
-                    plainSelect.getFromItem().accept(
-                            new TableNameReplaceVisitor(tableName, new HashSet<>(withNameList)));
+                    if (Objects.nonNull(plainSelect.getFromItem())) {
+                        plainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName,
+                                new HashSet<>(withNameList)));
+                    }
                 }
             });
             replaceJoins(painSelect, tableName, withNameList);
@@ -672,11 +674,13 @@ public class SqlReplaceHelper {
 
         List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
         for (PlainSelect plainSelect : plainSelects) {
-            Table table = (Table) plainSelect.getFromItem();
-            if (table.getName().equals(tableName)) {
-                replacePlainSelectByExpr(plainSelect, replace);
-                if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
-                    SqlSelectHelper.addMissingGroupby(plainSelect);
+            if (Objects.nonNull(plainSelect.getFromItem())) {
+                Table table = (Table) plainSelect.getFromItem();
+                if (table.getName().equals(tableName)) {
+                    replacePlainSelectByExpr(plainSelect, replace);
+                    if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
+                        SqlSelectHelper.addMissingGroupby(plainSelect);
+                    }
                 }
             }
         }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java

@@ -15,7 +15,7 @@ public enum DataType {
     SQLSERVER("sqlserver", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"",
             "\"", "\""),
 
-    H2("h2", "h2", "org.h2.Driver", "`", "`", "\"", "\""),
+    H2("H2", "h2", "org.h2.Driver", "`", "`", "\"", "\""),
 
     PHOENIX("phoenix", "hbase phoenix", "org.apache.phoenix.jdbc.PhoenixDriver", "", "", "\"",
             "\""),

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java

@@ -32,10 +32,11 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
     private static final String INSTRUCTION = ""
             + "#Role: You are a senior data engineer experienced in writing SQL."
             + "\n#Task: Your will be provided with a user question and the SQL written by a junior engineer,"
-            + "please take a review and help correct it if necessary." + "\n#Rules: "
-            + "\n1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),sql=(corrected sql if NEGATIVE; empty string if POSITIVE)`."
-            + "\n2.NO NEED to check date filters as the junior engineer seldom makes mistakes in this regard."
-            + "\n3.SQL columns and values must be mentioned in the `#Schema`."
+            + "please take a review and help correct it if necessary."
+            + "\n#Rules: "
+            + "1.ALWAYS specify time range using `>`,`<`,`>=`,`<=` operator."
+            + "2.DO NOT calculate date range using functions."
+            + "3.SQL columns and values must be mentioned in the `#Schema`."
             + "\n#Question:{{question}} #Schema:{{schema}} #InputSQL:{{sql}} #Response:";
 
     public LLMSqlCorrector() {
@@ -46,10 +47,10 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
     @Data
     @ToString
     static class SemanticSql {
-        @Description("positive or negative opinion about the sql")
+        @Description("either positive or negative")
         private String opinion;
 
-        @Description("corrected sql")
+        @Description("corrected sql if negative")
         private String sql;
     }
 
@@ -76,7 +77,7 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
                 semanticParseInfo, chatApp.getPrompt(), exemplar);
         SemanticSql s2Sql = extractor.generateSemanticSql(prompt.toUserMessage().singleText());
         keyPipelineLog.info("LLMSqlCorrector modelReq:\n{} \nmodelResp:\n{}", prompt.text(), s2Sql);
-        if ("NEGATIVE".equals(s2Sql.getOpinion()) && StringUtils.isNotBlank(s2Sql.getSql())) {
+        if ("NEGATIVE".equalsIgnoreCase(s2Sql.getOpinion()) && StringUtils.isNotBlank(s2Sql.getSql())) {
             semanticParseInfo.getSqlInfo().setCorrectedS2SQL(s2Sql.getSql());
         }
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java

@@ -29,6 +29,7 @@ public class OntologyQuery {
     private List<ColumnOrder> order;
     private boolean nativeQuery = true;
     private AggOption aggOption = AggOption.NATIVE;
+    private String sql;
 
     public Set<ModelResp> getModels() {
         return modelMap.values().stream().collect(Collectors.toSet());

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.core.pojo;
 
+import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
@@ -15,7 +16,7 @@ public class QueryStatement {
     private StructQuery structQuery;
     private SqlQuery sqlQuery;
     private OntologyQuery ontologyQuery;
-    private Integer status = 0;
+    private QueryState status = QueryState.SUCCESS;
     private Boolean isS2SQL = false;
     private Boolean enableOptimize = true;
     private Triple<String, String, String> minMaxTime;

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.translator;
 
 import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
 import com.tencent.supersonic.common.pojo.enums.EngineType;
+import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.pojo.SqlQuery;
@@ -15,82 +16,77 @@ import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 @Component
 @Slf4j
 public class DefaultSemanticTranslator implements SemanticTranslator {
 
-    public void translate(QueryStatement queryStatement) {
+    public void translate(QueryStatement queryStatement) throws Exception {
         if (queryStatement.isTranslated()) {
             return;
         }
-        try {
-            for (QueryParser parser : ComponentFactory.getQueryParsers()) {
-                if (parser.accept(queryStatement)) {
-                    log.debug("QueryConverter accept [{}]", parser.getClass().getName());
-                    parser.parse(queryStatement);
-                    if (queryStatement.getStatus() != 0) {
-                        break;
-                    }
+        for (QueryParser parser : ComponentFactory.getQueryParsers()) {
+            if (parser.accept(queryStatement)) {
+                log.debug("QueryConverter accept [{}]", parser.getClass().getName());
+                parser.parse(queryStatement);
+                if (!queryStatement.getStatus().equals(QueryState.SUCCESS)) {
+                    break;
                 }
             }
-            mergeOntologyQuery(queryStatement);
-
-            if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) {
-                queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql());
-            }
-            if (StringUtils.isBlank(queryStatement.getSql())) {
-                throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
-            }
-
-            for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) {
-                if (optimizer.accept(queryStatement)) {
-                    optimizer.rewrite(queryStatement);
-                }
-            }
-            log.info("translated query SQL: [{}]",
-                    StringUtils.normalizeSpace(queryStatement.getSql()));
-        } catch (Exception e) {
-            queryStatement.setErrMsg(e.getMessage());
-            log.error("Failed to translate query [{}]", e.getMessage(), e);
         }
+        mergeOntologyQuery(queryStatement);
+
+        if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) {
+            queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql());
+        }
+        if (StringUtils.isBlank(queryStatement.getSql())) {
+            throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
+        }
+
+        for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) {
+            if (optimizer.accept(queryStatement)) {
+                optimizer.rewrite(queryStatement);
+            }
+        }
+        log.info("translated query SQL: [{}]", StringUtils.normalizeSpace(queryStatement.getSql()));
     }
 
     private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
         OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
         log.info("parse with ontology: [{}]", ontologyQuery);
 
-        if (!queryStatement.isOk()) {
+        if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) {
             throw new Exception(String.format("parse ontology table [%s] error [%s]",
                     queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg()));
         }
 
         SqlQuery sqlQuery = queryStatement.getSqlQuery();
-        String ontologyQuerySql = sqlQuery.getSql();
+        String ontologyOuterSql = sqlQuery.getSql();
         String ontologyInnerTable = sqlQuery.getTable();
-        String ontologyInnerSql = queryStatement.getSql();
+        String ontologyInnerSql = ontologyQuery.getSql();
 
         List<Pair<String, String>> tables = new ArrayList<>();
         tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql));
         String finalSql = null;
         if (sqlQuery.isSupportWith()) {
             EngineType engineType = queryStatement.getOntology().getDatabaseType();
-            if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) {
+            if (!SqlMergeWithUtils.hasWith(engineType, ontologyOuterSql)) {
                 finalSql = "with " + tables.stream()
                         .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
-                        .collect(Collectors.joining(",")) + "\n" + ontologyQuerySql;
+                        .collect(Collectors.joining(",")) + "\n" + ontologyOuterSql;
             } else {
                 List<String> withTableList =
                         tables.stream().map(Pair::getLeft).collect(Collectors.toList());
                 List<String> withSqlList =
                         tables.stream().map(Pair::getRight).collect(Collectors.toList());
-                finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, withSqlList,
+                finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyOuterSql, withSqlList,
                         withTableList);
             }
         } else {
             for (Pair<String, String> tb : tables) {
-                finalSql = StringUtils.replace(ontologyQuerySql, tb.getLeft(),
+                finalSql = StringUtils.replace(ontologyOuterSql, tb.getLeft(),
                         "(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()),
                         -1);
             }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java

@@ -8,5 +8,5 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
  */
 public interface SemanticTranslator {
 
-    void translate(QueryStatement queryStatement);
+    void translate(QueryStatement queryStatement) throws Exception;
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java

@@ -32,7 +32,7 @@ public class OntologyQueryParser implements QueryParser {
                 .build();
         SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema);
         String sql = sqlBuilder.buildOntologySql(queryStatement);
-        queryStatement.setSql(sql);
+        queryStatement.getOntologyQuery().setSql(sql);
     }
 
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.core.translator.parser;
 
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
@@ -10,6 +11,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import com.tencent.supersonic.headless.core.pojo.Ontology;
 import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -48,9 +50,13 @@ public class SqlQueryParser implements QueryParser {
         // check if there are fields not matched with any metric or dimension
         if (queryFields.size() > ontologyQuery.getMetrics().size()
                 + ontologyQuery.getDimensions().size()) {
-            queryStatement
-                    .setErrMsg("There are fields in the SQL not matched with any semantic column.");
-            queryStatement.setStatus(1);
+            List<String> semanticFields = Lists.newArrayList();
+            ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
+            ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));
+            String errMsg = String.format("Querying columns[%s] not matched with semantic fields[%s].",
+                            queryFields, semanticFields);
+            queryStatement.setErrMsg(errMsg);
+            queryStatement.setStatus(QueryState.INVALID);
             return;
         }
         queryStatement.setOntologyQuery(ontologyQuery);

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -337,7 +337,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
         List<QueryStatement> queryStatements = new ArrayList<>();
         for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
             QueryStatement queryStatement = buildStructQueryStatement(queryStructReq);
-            semanticTranslator.translate(queryStatement);
+            try {
+                semanticTranslator.translate(queryStatement);
+            } catch (Exception e) {
+                log.warn("Failed to translate for semantic query " + queryStructReq);
+            }
             queryStatements.add(queryStatement);
         }
         log.info("Union multiple query statements:{}", queryStatements);

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java

@@ -152,7 +152,7 @@ public class ChatWorkflowEngine {
                         StringUtils.normalizeSpace(parseInfo.getSqlInfo().getCorrectedS2SQL()),
                         StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL()));
             } catch (Exception e) {
-                log.warn("get sql info failed:{}", parseInfo, e);
+                log.warn("get sql info failed:{}", e);
                 errorMsg.add(String.format("S2SQL:%s %s", parseInfo.getSqlInfo().getParsedS2SQL(),
                         e.getMessage()));
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java

@@ -119,10 +119,17 @@ public class QueryUtils {
         if (StringUtils.isBlank(type)) {
             return false;
         }
-        return type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
-                || type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")
-                || type.equalsIgnoreCase("numeric") || type.toLowerCase().startsWith("decimal")
-                || type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int");
+        return type.equalsIgnoreCase("int")
+                || type.equalsIgnoreCase("bigint")
+                || type.equalsIgnoreCase("tinyint")
+                || type.equalsIgnoreCase("smallint")
+                || type.equalsIgnoreCase("float")
+                || type.equalsIgnoreCase("double")
+                || type.equalsIgnoreCase("real")
+                || type.equalsIgnoreCase("numeric")
+                || type.toLowerCase().startsWith("decimal")
+                || type.toLowerCase().startsWith("uint")
+                || type.toLowerCase().startsWith("int");
     }
 
     private String getName(String nameEn) {

launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java

@@ -121,7 +121,13 @@ public abstract class S2BaseDemo implements CommandLineRunner {
         DatabaseReq databaseReq = new DatabaseReq();
         databaseReq.setName("S2数据库DEMO");
         databaseReq.setDescription("样例数据库实例仅用于体验");
-        databaseReq.setType(DataType.POSTGRESQL.getFeature());
+        databaseReq.setType(DataType.H2.getFeature());
+        String profile = System.getProperty("spring.profiles.active");
+        if ("postgres".equalsIgnoreCase(profile)) {
+            databaseReq.setType(DataType.POSTGRESQL.getFeature());
+        } else if ("mysql".equalsIgnoreCase(profile)) {
+            databaseReq.setType(DataType.MYSQL.getFeature());
+        }
         databaseReq.setUrl(url);
         databaseReq.setUsername(dataSourceProperties.getUsername());
         databaseReq