[improvement][headless]Expression replacement logic supports more complex sql.

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java

@@ -627,7 +627,8 @@ public class SqlReplaceHelper {
         return expr;
     }
 
-    public static String replaceSqlByExpression(String sql, Map<String, String> replace) {
+    public static String replaceSqlByExpression(String tableName, String sql,
+            Map<String, String> replace) {
         Select selectStatement = SqlSelectHelper.getSelect(sql);
         List<PlainSelect> plainSelectList = new ArrayList<>();
         if (selectStatement instanceof PlainSelect) {
@@ -636,9 +637,8 @@ public class SqlReplaceHelper {
                 selectStatement.getWithItemsList().forEach(withItem -> {
                     plainSelectList.add(withItem.getSelect().getPlainSelect());
                 });
-            } else {
-                plainSelectList.add((PlainSelect) selectStatement);
             }
+            plainSelectList.add((PlainSelect) selectStatement);
         } else if (selectStatement instanceof SetOperationList) {
             SetOperationList setOperationList = (SetOperationList) selectStatement;
             if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
@@ -672,9 +672,12 @@ public class SqlReplaceHelper {
 
         List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
         for (PlainSelect plainSelect : plainSelects) {
-            replacePlainSelectByExpr(plainSelect, replace);
+            Table table = (Table) plainSelect.getFromItem();
-            if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
+            if (table.getName().equals(tableName)) {
-                SqlSelectHelper.addMissingGroupby(plainSelect);
+                replacePlainSelectByExpr(plainSelect, replace);
+                if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
+                    SqlSelectHelper.addMissingGroupby(plainSelect);
+                }
             }
         }
         return selectStatement.toString();

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java

@@ -1,12 +1,16 @@
 package com.tencent.supersonic.headless.api.pojo;
 
 import com.google.common.base.Objects;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 import java.util.List;
 
 @Data
+@Builder
+@AllArgsConstructor
 @NoArgsConstructor
 public class MetaFilter {
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java

@@ -30,10 +30,6 @@ public class DataSetResp extends SchemaItem {
 
     private QueryConfig queryConfig = new QueryConfig();
 
-    private List<TagItem> allMetrics = new ArrayList<>();
-
-    private List<TagItem> allDimensions = new ArrayList<>();
-
     public List<Long> metricIds() {
         return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics)
                 .flatMap(Collection::stream).collect(Collectors.toList());

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java

@@ -22,7 +22,6 @@ public class SemanticSchemaResp {
     private SchemaType schemaType;
     private List<MetricSchemaResp> metrics = Lists.newArrayList();
     private List<DimSchemaResp> dimensions = Lists.newArrayList();
-    private List<TagResp> tags = Lists.newArrayList();
     private List<ModelRela> modelRelas = Lists.newArrayList();
     private List<ModelResp> modelResps = Lists.newArrayList();
     private DataSetResp dataSetResp;

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java

@@ -35,7 +35,8 @@ public class LLMSqlParser implements SemanticParser {
             if (dataSetId == null) {
                 return;
             }
-            log.info("try generating query statement for dataSetId:{}", dataSetId);
+            log.info("try generating query statement for query:{}, dataSetId:{}",
+                    queryCtx.getRequest().getQueryText(), dataSetId);
 
             // 3.invoke LLM service to do parsing.
             tryParse(queryCtx, dataSetId);

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DimExpressionParser.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.translator.parser;
 
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
+import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -40,7 +41,9 @@ public class DimExpressionParser implements QueryParser {
 
         Map<String, String> bizName2Expr = getDimensionExpressions(semanticSchema, ontologyQuery);
         if (!CollectionUtils.isEmpty(bizName2Expr)) {
-            String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr);
+            String sql = SqlReplaceHelper.replaceSqlByExpression(
+                    Constants.TABLE_PREFIX + queryStatement.getDataSetId(), sqlQuery.getSql(),
+                    bizName2Expr);
             sqlQuery.setSql(sql);
         }
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricExpressionParser.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.translator.parser;
 
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
+import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -39,7 +40,9 @@ public class MetricExpressionParser implements QueryParser {
 
         Map<String, String> bizName2Expr = getMetricExpressions(semanticSchema, ontologyQuery);
         if (!CollectionUtils.isEmpty(bizName2Expr)) {
-            String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr);
+            String sql = SqlReplaceHelper.replaceSqlByExpression(
+                    Constants.TABLE_PREFIX + queryStatement.getDataSetId(), sqlQuery.getSql(),
+                    bizName2Expr);
             sqlQuery.setSql(sql);
         }
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java

@@ -15,24 +15,14 @@ import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
 import com.tencent.supersonic.headless.api.pojo.MetaFilter;
 import com.tencent.supersonic.headless.api.pojo.QueryConfig;
-import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
+import com.tencent.supersonic.headless.api.pojo.request.*;
-import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
-import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
 import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
-import com.tencent.supersonic.headless.api.pojo.response.TagItem;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper;
-import com.tencent.supersonic.headless.server.service.DataSetService;
+import com.tencent.supersonic.headless.server.service.*;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.DomainService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -42,14 +32,7 @@ import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.Arrays;
+import java.util.*;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -69,10 +52,6 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
     @Autowired
     private MetricService metricService;
 
-    @Lazy
-    @Autowired
-    private TagMetaService tagMetaService;
-
     @Override
     public DataSetResp save(DataSetReq dataSetReq, User user) {
         dataSetReq.createdBy(user.getName());
@@ -199,13 +178,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
         dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList()
                 : Arrays.asList(dataSetDO.getAdminOrg().split(",")));
         dataSetResp.setTypeEnum(TypeEnums.DATASET);
-        List<TagItem> dimensionItems =
-                tagMetaService.getTagItems(dataSetResp.dimensionIds(), TagDefineType.DIMENSION);
-        dataSetResp.setAllDimensions(dimensionItems);
 
-        List<TagItem> metricItems =
-                tagMetaService.getTagItems(dataSetResp.metricIds(), TagDefineType.METRIC);
-        dataSetResp.setAllMetrics(metricItems);
         return dataSetResp;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java

@@ -11,30 +11,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
+import com.tencent.supersonic.headless.api.pojo.*;
-import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
-import com.tencent.supersonic.headless.api.pojo.MetaFilter;
-import com.tencent.supersonic.headless.api.pojo.SchemaItem;
-import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
 import com.tencent.supersonic.headless.api.pojo.enums.SchemaType;
 import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
 import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
+import com.tencent.supersonic.headless.api.pojo.response.*;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
-import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
-import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
-import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
-import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.TagResp;
-import com.tencent.supersonic.headless.api.pojo.response.TermResp;
 import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
 import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
 import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
@@ -43,16 +25,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
 import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
 import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
-import com.tencent.supersonic.headless.server.service.DataSetService;
+import com.tencent.supersonic.headless.server.service.*;
-import com.tencent.supersonic.headless.server.service.DatabaseService;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.DomainService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.ModelRelaService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.SchemaService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
-import com.tencent.supersonic.headless.server.service.TermService;
 import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder;
 import com.tencent.supersonic.headless.server.utils.DimensionConverter;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -64,14 +37,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -97,7 +63,6 @@ public class SchemaServiceImpl implements SchemaService {
     private final DomainService domainService;
     private final DataSetService dataSetService;
     private final ModelRelaService modelRelaService;
-    private final TagMetaService tagService;
     private final TermService termService;
     private final DatabaseService databaseService;
 
@@ -106,8 +71,8 @@ public class SchemaServiceImpl implements SchemaService {
 
     public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService,
             MetricService metricService, DomainService domainService, DataSetService dataSetService,
-            ModelRelaService modelRelaService, StatUtils statUtils, TagMetaService tagService,
+            ModelRelaService modelRelaService, StatUtils statUtils, TermService termService,
-            TermService termService, DatabaseService databaseService) {
+            DatabaseService databaseService) {
         this.modelService = modelService;
         this.dimensionService = dimensionService;
         this.metricService = metricService;
@@ -115,7 +80,6 @@ public class SchemaServiceImpl implements SchemaService {
         this.dataSetService = dataSetService;
         this.modelRelaService = modelRelaService;
         this.statUtils = statUtils;
-        this.tagService = tagService;
         this.termService = termService;
         this.databaseService = databaseService;
     }
@@ -383,6 +347,7 @@ public class SchemaServiceImpl implements SchemaService {
             DataSetSchemaResp dataSetSchemaResp =
                     fetchDataSetSchema(schemaFilterReq.getDataSetId());
             BeanUtils.copyProperties(dataSetSchemaResp, semanticSchemaResp);
+            semanticSchemaResp.setDataSetResp(dataSetSchemaResp);
             List<Long> modelIds = dataSetSchemaResp.getAllModels();
             MetaFilter metaFilter = new MetaFilter();
             metaFilter.setIds(modelIds);
@@ -408,13 +373,7 @@ public class SchemaServiceImpl implements SchemaService {
                     modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
             semanticSchemaResp.setSchemaType(SchemaType.MODEL);
         }
-        if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
+
-            // add tag info
-            TagFilter tagFilter = new TagFilter();
-            tagFilter.setModelIds(semanticSchemaResp.getModelIds());
-            List<TagResp> tagResps = tagService.getTags(tagFilter);
-            semanticSchemaResp.setTags(tagResps);
-        }
         if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
             DatabaseResp databaseResp =
                     modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -17,13 +17,11 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
 import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.xkzhangsan.time.utils.CollectionUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
@@ -60,16 +58,13 @@ public class DictUtils {
     private final MetricService metricService;
     private final SemanticLayerService queryService;
     private final ModelService modelService;
-    private final TagMetaService tagMetaService;
 
     public DictUtils(DimensionService dimensionService, MetricService metricService,
-            SemanticLayerService queryService, ModelService modelService,
+            SemanticLayerService queryService, ModelService modelService) {
-            @Lazy TagMetaService tagMetaService) {
         this.dimensionService = dimensionService;
         this.metricService = metricService;
         this.queryService = queryService;
         this.modelService = modelService;
-        this.tagMetaService = tagMetaService;
     }
 
     public String fetchDictFileName(DictItemResp dictItemResp) {
@@ -133,11 +128,6 @@ public class DictUtils {
             dictItemResp.setModelId(dimension.getModelId());
             dictItemResp.setBizName(dimension.getBizName());
         }
-        if (TypeEnums.TAG.equals(TypeEnums.valueOf(dictConfDO.getType()))) {
-            TagResp tagResp = tagMetaService.getTag(dictConfDO.getItemId(), User.getDefaultUser());
-            dictItemResp.setModelId(tagResp.getModelId());
-            dictItemResp.setBizName(tagResp.getBizName());
-        }
 
         return dictItemResp;
     }

launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslatorTest.java

@@ -111,4 +111,18 @@ public class TranslatorTest extends BaseTest {
         executeSql(explain.getQuerySQL());
     }
 
+    @Test
+    @SetSystemProperty(key = "s2.test", value = "true")
+    public void testSql_subquery() throws Exception {
+        String sql = new String(
+                Files.readAllBytes(
+                        Paths.get(ClassLoader.getSystemResource("sql/testSubquery.sql").toURI())),
+                StandardCharsets.UTF_8);
+        SemanticTranslateResp explain = semanticLayerService
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+        assertNotNull(explain);
+        assertNotNull(explain.getQuerySQL());
+        executeSql(explain.getQuerySQL());
+    }
+
 }

launchers/standalone/src/test/resources/sql/testSubquery.sql

@@ -0,0 +1,27 @@
+WITH
+  _average_stay_duration_ AS (
+    SELECT
+      AVG(停留时长) AS _avg_duration_
+    FROM
+      超音数数据集
+  )
+SELECT
+  用户名,
+  SUM(停留时长) AS _total_stay_duration_
+FROM
+  超音数数据集
+GROUP BY
+  用户名
+HAVING
+  SUM(停留时长) > (
+    SELECT
+      _avg_duration_ * 1.5
+    FROM
+      _average_stay_duration_
+  )
+  OR SUM(停留时长) < (
+    SELECT
+      _avg_duration_ * 0.5
+    FROM
+      _average_stay_duration_
+  )