Merge 668f872743 into a3293e6788

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java

@@ -225,7 +225,7 @@ public class SqlSelectHelper {
     public static Select getSelect(String sql) {
         Statement statement = null;
         try {
-            statement = CCJSqlParserUtil.parse(sql);
+            statement = CCJSqlParserUtil.parse(sql, parser -> parser.withTimeOut(20000));
         } catch (JSQLParserException e) {
             log.error("parse error, sql:{}", sql, e);
             throw new RuntimeException(e);

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java

@@ -23,12 +23,16 @@ public class Measure {
 
     private String alias;
 
-    public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) {
+    private String unit;
+
+    public Measure(String name, String bizName, String expr, String agg, String unit,
+            Integer isCreateMetric) {
         this.name = name;
         this.agg = agg;
         this.isCreateMetric = isCreateMetric;
         this.bizName = bizName;
         this.expr = expr;
+        this.unit = unit;
     }
 
     public Measure(String name, String bizName, String agg, Integer isCreateMetric) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -49,27 +49,4 @@ public class ModelDetail {
                 .collect(Collectors.toList());
     }
 
-    public List<Field> getFields() {
-        if (!CollectionUtils.isEmpty(fields)) {
-            return fields;
-        }
-        List<Field> fieldList = Lists.newArrayList();
-        // Compatible with older versions
-        if (!CollectionUtils.isEmpty(identifiers)) {
-            fieldList.addAll(identifiers.stream()
-                    .map(identify -> Field.builder().fieldName(identify.getFieldName()).build())
-                    .collect(Collectors.toSet()));
-        }
-        if (!CollectionUtils.isEmpty(dimensions)) {
-            fieldList.addAll(dimensions.stream()
-                    .map(dim -> Field.builder().fieldName(dim.getFieldName()).build())
-                    .collect(Collectors.toSet()));
-        }
-        if (!CollectionUtils.isEmpty(measures)) {
-            fieldList.addAll(measures.stream()
-                    .map(measure -> Field.builder().fieldName(measure.getFieldName()).build())
-                    .collect(Collectors.toSet()));
-        }
-        return fieldList;
-    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java

@@ -14,11 +14,11 @@ public class ModelSchema {
 
     private String description;
 
-    private List<ColumnSchema> columnSchemas;
+    private List<SemanticColumn> semanticColumns;
 
     @JsonIgnore
-    public ColumnSchema getColumnByName(String columnName) {
-        for (ColumnSchema fieldSchema : columnSchemas) {
+    public SemanticColumn getColumnByName(String columnName) {
+        for (SemanticColumn fieldSchema : semanticColumns) {
             if (fieldSchema.getColumnName().equalsIgnoreCase(columnName)) {
                 return fieldSchema;
             }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticColumn.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
 import lombok.Data;
 
 @Data
-public class ColumnSchema {
+public class SemanticColumn {
 
     private String columnName;
 
@@ -19,4 +19,8 @@ public class ColumnSchema {
 
     private String name;
 
+    private String expr;
+
+    private String unit;
+
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PrestoAdaptor.java

@@ -83,6 +83,7 @@ public class PrestoAdaptor extends BaseDbAdaptor {
 
     @Override
     public String rewriteSql(String sql) {
+        sql = sql.replaceAll("`", "\"");
         return sql;
     }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -100,6 +100,7 @@ public class SemanticSchemaManager {
         modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
         modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
         modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));
+        modelDetail.getFields().addAll(d.getFields());
 
         return dataModel;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/LLMSemanticModeller.java

@@ -75,6 +75,7 @@ public class LLMSemanticModeller implements SemanticModeller {
         if (!chatApp.isPresent() || !chatApp.get().isEnable()) {
             return;
         }
+
         List<DbSchema> otherDbSchema = getOtherDbSchema(dbSchema, dbSchemas);
         ModelSchemaExtractor extractor =
                 AiServices.create(ModelSchemaExtractor.class, getChatModel(modelBuildReq));

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java

@@ -1,9 +1,9 @@
 package com.tencent.supersonic.headless.server.modeller;
 
-import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.DbSchema;
 import com.tencent.supersonic.headless.api.pojo.ModelSchema;
+import com.tencent.supersonic.headless.api.pojo.SemanticColumn;
 import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
 import lombok.extern.slf4j.Slf4j;
 
@@ -16,19 +16,20 @@ public class RuleSemanticModeller implements SemanticModeller {
     @Override
     public void build(DbSchema dbSchema, List<DbSchema> dbSchemas, ModelSchema modelSchema,
             ModelBuildReq modelBuildReq) {
-        List<ColumnSchema> columnSchemas =
+        List<SemanticColumn> semanticColumns =
                 dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
-        modelSchema.setColumnSchemas(columnSchemas);
+        modelSchema.setSemanticColumns(semanticColumns);
     }
 
-    private ColumnSchema convert(DBColumn dbColumn) {
-        ColumnSchema columnSchema = new ColumnSchema();
-        columnSchema.setName(dbColumn.getColumnName());
-        columnSchema.setColumnName(dbColumn.getColumnName());
-        columnSchema.setComment(dbColumn.getComment());
-        columnSchema.setDataType(dbColumn.getDataType());
-        columnSchema.setFiledType(dbColumn.getFieldType());
-        return columnSchema;
+    private SemanticColumn convert(DBColumn dbColumn) {
+        SemanticColumn semanticColumn = new SemanticColumn();
+        semanticColumn.setName(dbColumn.getColumnName());
+        semanticColumn.setColumnName(dbColumn.getColumnName());
+        semanticColumn.setExpr(dbColumn.getColumnName());
+        semanticColumn.setComment(dbColumn.getComment());
+        semanticColumn.setDataType(dbColumn.getDataType());
+        semanticColumn.setFiledType(dbColumn.getFieldType());
+        return semanticColumn;
     }
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java

@@ -7,6 +7,9 @@ import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
 
 import java.util.List;
 
+/**
+ * A semantic modeler builds semantic-layer schemas from database-layer schemas.
+ */
 public interface SemanticModeller {
 
     void build(DbSchema dbSchema, List<DbSchema> otherDbSchema, ModelSchema modelSchema,

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -277,8 +277,8 @@ public class ModelServiceImpl implements ModelService {
         dimensionService.createDimensionBatch(dimensionReqs, user);
     }
 
-    private void batchCreateMetric(ModelDO datasourceDO, User user) throws Exception {
-        List<MetricReq> metricReqs = ModelConverter.convertMetricList(datasourceDO);
+    private void batchCreateMetric(ModelDO modelDO, User user) throws Exception {
+        List<MetricReq> metricReqs = ModelConverter.convertMetricList(modelDO);
         metricService.createMetricBatch(metricReqs, user);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -425,8 +425,12 @@ public class DictUtils {
                 .format(DateTimeFormatter.ofPattern(format));
         String end = LocalDate.now().minusDays(itemValueDateEnd)
                 .format(DateTimeFormatter.ofPattern(format));
-        return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
-                dateConf.getDateField(), end);
+        if (Objects.nonNull(dateConf)) {
+            return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
+                    dateConf.getDateField(), end);
+        } else {
+            return String.format("( %s >= '%s' and %s <= '%s' )", "dt", start, "dt", end);
+        }
     }
 
     private String generateDictDateFilter(DictItemResp dictItemResp) {
@@ -440,7 +444,7 @@ public class DictUtils {
         }
         // 未进行设置
         if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
-            return defaultDateFilter(config.getDateConf());
+            return defaultDateFilter(null);
         }
         // 全表扫描
         if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -157,23 +157,28 @@ public class ModelConverter {
             modelDetail.setQueryType(ModelDefineType.TABLE_QUERY.getName());
             modelDetail.setTableQuery(String.format("%s.%s", modelBuildReq.getDb(), tableName));
         }
-        for (ColumnSchema columnSchema : modelSchema.getColumnSchemas()) {
-            FieldType fieldType = columnSchema.getFiledType();
+        List<Field> fields = new ArrayList<>();
+        for (SemanticColumn semanticColumn : modelSchema.getSemanticColumns()) {
+            FieldType fieldType = semanticColumn.getFiledType();
+            fields.add(new Field(semanticColumn.getName(), semanticColumn.getDataType()));
+
             if (getIdentifyType(fieldType) != null) {
-                Identify identify = new Identify(columnSchema.getName(),
-                        getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
+                Identify identify = new Identify(semanticColumn.getName(),
+                        getIdentifyType(fieldType).name(), semanticColumn.getColumnName(), 1);
                 modelDetail.getIdentifiers().add(identify);
             } else if (FieldType.measure.equals(fieldType)) {
-                Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
-                        columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
+                Measure measure = new Measure(semanticColumn.getName(),
+                        semanticColumn.getColumnName(), semanticColumn.getExpr(),
+                        semanticColumn.getAgg().getOperator(), semanticColumn.getUnit(), 1);
                 modelDetail.getMeasures().add(measure);
             } else {
-                Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
-                        columnSchema.getColumnName(),
-                        DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
+                Dimension dim = new Dimension(semanticColumn.getName(),
+                        semanticColumn.getColumnName(), semanticColumn.getExpr(),
+                        DimensionType.valueOf(semanticColumn.getFiledType().name()), 1);
                 modelDetail.getDimensions().add(dim);
             }
         }
+        modelDetail.setFields(fields);
         modelReq.setModelDetail(modelDetail);
         return modelReq;
     }

launchers/standalone/src/main/resources/config.update/sql-update-mysql.sql

@@ -410,4 +410,7 @@ ALTER TABLE s2_query_stat_info RENAME COLUMN `sql` TO `query_sql`;
 --20250224
 ALTER TABLE s2_agent add column `admin_org` varchar(3000) DEFAULT NULL COMMENT '管理员组织';
 ALTER TABLE s2_agent add column `view_org` varchar(3000) DEFAULT NULL COMMENT '可用组织';
-ALTER TABLE s2_agent add column `is_open` tinyint DEFAULT NULL COMMENT '是否公开';
+ALTER TABLE s2_agent add column `is_open` tinyint DEFAULT NULL COMMENT '是否公开';
+
+--20250309
+ALTER TABLE s2_model_rela alter column join_condition type text;

launchers/standalone/src/main/resources/db/schema-h2.sql

@@ -268,7 +268,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
     from_model_id    BIGINT,
     to_model_id      BIGINT,
     join_type       VARCHAR(255),
-    join_condition  VARCHAR(255),
+    join_condition  TEXT,
     PRIMARY KEY (`id`)
 );
 

launchers/standalone/src/main/resources/db/schema-mysql.sql

@@ -404,7 +404,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
     from_model_id    bigint,
     to_model_id      bigint,
     join_type       VARCHAR(255),
-    join_condition  VARCHAR(255)
+    join_condition  text
 )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 CREATE TABLE IF NOT EXISTS `s2_collect` (

launchers/standalone/src/main/resources/db/schema-postgres.sql

@@ -465,7 +465,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela (
     from_model_id bigint,
     to_model_id bigint,
     join_type VARCHAR(255),
-    join_condition VARCHAR(255)
+    join_condition text
 );
 
 CREATE TABLE IF NOT EXISTS s2_collect (

launchers/standalone/src/test/java/com/tencent/supersonic/headless/SemanticModellerTest.java

@@ -39,14 +39,14 @@ public class SemanticModellerTest extends BaseTest {
         Map<String, ModelSchema> modelSchemaMap = modelService.buildModelSchema(modelSchemaReq);
 
         ModelSchema userModelSchema = modelSchemaMap.get("s2_user_department");
-        Assertions.assertEquals(2, userModelSchema.getColumnSchemas().size());
+        Assertions.assertEquals(2, userModelSchema.getSemanticColumns().size());
         Assertions.assertEquals(FieldType.primary_key,
                 userModelSchema.getColumnByName("user_name").getFiledType());
         Assertions.assertEquals(FieldType.categorical,
                 userModelSchema.getColumnByName("department").getFiledType());
 
         ModelSchema stayTimeModelSchema = modelSchemaMap.get("s2_stay_time_statis");
-        Assertions.assertEquals(4, stayTimeModelSchema.getColumnSchemas().size());
+        Assertions.assertEquals(4, stayTimeModelSchema.getSemanticColumns().size());
         Assertions.assertEquals(FieldType.foreign_key,
                 stayTimeModelSchema.getColumnByName("user_name").getFiledType());
         Assertions.assertEquals(FieldType.partition_time,
@@ -72,7 +72,7 @@ public class SemanticModellerTest extends BaseTest {
         Map<String, ModelSchema> modelSchemaMap = modelService.buildModelSchema(modelSchemaReq);
 
         ModelSchema pvModelSchema = modelSchemaMap.values().iterator().next();
-        Assertions.assertEquals(5, pvModelSchema.getColumnSchemas().size());
+        Assertions.assertEquals(5, pvModelSchema.getSemanticColumns().size());
         Assertions.assertEquals(FieldType.partition_time,
                 pvModelSchema.getColumnByName("imp_date").getFiledType());
         Assertions.assertEquals(FieldType.categorical,

pom.xml

@@ -32,7 +32,7 @@
         <maven.compiler.source>21</maven.compiler.source>
         <maven.compiler.target>21</maven.compiler.target>
         <file.encoding>UTF-8</file.encoding>
-        <jsqlparser.version>4.7</jsqlparser.version>
+        <jsqlparser.version>4.9</jsqlparser.version>
         <pagehelper.version>6.1.0</pagehelper.version>
         <pagehelper.spring.version>2.1.0</pagehelper.spring.version>
         <mybatis.version>3.5.3</mybatis.version>