(improvement)(Headless) corrector supports subselect sql and opt BenchMarkDemoDataLoader (#1023)

* [improvement] corrector and QueryReqConverter  support subselect

* [improvement] corrector and QueryReqConverter  support subselect

* [improvement] adapt BenchMarkDemoDataLoader

* [improvement] adapt checkstyle

---------

Co-authored-by: zuopengge

common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java

@@ -5,6 +5,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.Function;
@@ -23,6 +25,7 @@ import net.sf.jsqlparser.statement.select.Select;
 import net.sf.jsqlparser.statement.select.SelectItem;
 import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
 import net.sf.jsqlparser.statement.select.SetOperationList;
+import net.sf.jsqlparser.statement.select.ParenthesedSelect;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
 
@@ -148,6 +151,24 @@ public class SqlAddHelper {
             return sql;
         }
         PlainSelect plainSelect = (PlainSelect) selectStatement;
+        List<String> chNameList = TimeDimensionEnum.getChNameList();
+        Boolean dateWhere = false;
+        for (String chName : chNameList) {
+            if (expression.toString().contains(chName)) {
+                dateWhere = true;
+            }
+        }
+        if (plainSelect.getFromItem() instanceof ParenthesedSelect && dateWhere) {
+            ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem();
+            PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect();
+            Expression subWhere = subPlainSelect.getWhere();
+            if (subWhere == null) {
+                subPlainSelect.setWhere(expression);
+            } else {
+                subPlainSelect.setWhere(new AndExpression(subWhere, expression));
+            }
+            return selectStatement.toString();
+        }
         Expression where = plainSelect.getWhere();
 
         if (where == null) {

common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java

@@ -397,12 +397,14 @@ public class SqlReplaceHelper {
         if (selectStatement instanceof PlainSelect) {
             PlainSelect plainSelect = (PlainSelect) selectStatement;
             replaceSingleTable(plainSelect, tableName);
+            replaceSubTable(plainSelect, tableName);
         } else if (selectStatement instanceof SetOperationList) {
             SetOperationList setOperationList = (SetOperationList) selectStatement;
             if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
                 setOperationList.getSelects().forEach(subSelectBody -> {
                     PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
                     replaceSingleTable(subPlainSelect, tableName);
+                    replaceSubTable(subPlainSelect, tableName);
                 });
             }
         }
@@ -410,6 +412,14 @@ public class SqlReplaceHelper {
         return selectStatement.toString();
     }
 
+    public static void replaceSubTable(PlainSelect plainSelect, String tableName) {
+        if (plainSelect.getFromItem() instanceof ParenthesedSelect) {
+            ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem();
+            PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect();
+            replaceSingleTable(subPlainSelect, tableName);
+        }
+    }
+
     public static void replaceSingleTable(PlainSelect plainSelect, String tableName) {
         // replace table name
         List<PlainSelect> plainSelects = new ArrayList<>();

launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java

@@ -25,6 +25,9 @@ import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
 import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
 import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
 import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
+import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
+import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
+import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.server.service.DomainService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelRelaService;
@@ -55,28 +58,31 @@ public class BenchMarkDemoDataLoader {
     private DataSetService viewService;
     @Autowired
     private MetricService metricService;
+    @Autowired
+    private ModelDemoDataLoader modelDemoDataLoader;
 
     public void doRun() {
         try {
-            addDomain();
+            DatabaseResp databaseResp = modelDemoDataLoader.tmpDatabaseResp;
-            addModel_1();
+            DomainResp s2Domain = addDomain();
-            addModel_2();
+            ModelResp genreModelResp = addModel_1(s2Domain, databaseResp);
-            addModel_3();
+            ModelResp artistModelResp = addModel_2(s2Domain, databaseResp);
-            addModel_4();
+            ModelResp filesModelResp = addModel_3(s2Domain, databaseResp);
-            addDataSet_1();
+            ModelResp songModelResp = addModel_4(s2Domain, databaseResp);
-            addModelRela_1();
+            addDataSet_1(s2Domain);
-            addModelRela_2();
+            addModelRela_1(s2Domain, genreModelResp, artistModelResp);
-            addModelRela_3();
+            addModelRela_2(s2Domain, filesModelResp, artistModelResp);
-            addModelRela_4();
+            addModelRela_3(s2Domain, songModelResp, artistModelResp);
-            addModelRela_5();
+            addModelRela_4(s2Domain, songModelResp, genreModelResp);
-            batchPushlishMetric();
+            addModelRela_5(s2Domain, songModelResp, filesModelResp);
+            //batchPushlishMetric();
         } catch (Exception e) {
             log.error("Failed to add bench mark demo data", e);
         }
 
     }
 
-    public void addDomain() {
+    public DomainResp addDomain() {
         DomainReq domainReq = new DomainReq();
         domainReq.setName("测评数据_音乐");
         domainReq.setBizName("music");
@@ -85,17 +91,16 @@ public class BenchMarkDemoDataLoader {
         domainReq.setViewOrgs(Collections.singletonList("1"));
         domainReq.setAdmins(Collections.singletonList("admin"));
         domainReq.setAdminOrgs(Collections.emptyList());
-        domainService.createDomain(domainReq, user);
+        return domainService.createDomain(domainReq, user);
     }
 
-    public void addModel_1() throws Exception {
+    public ModelResp addModel_1(DomainResp s2Domain, DatabaseResp s2Database) throws Exception {
         ModelReq modelReq = new ModelReq();
-        modelReq.setDomainId(3L);
+        modelReq.setDomainId(s2Domain.getId());
         modelReq.setName("艺术类型");
         modelReq.setBizName("genre");
         modelReq.setDescription("艺术类型");
-        modelReq.setDatabaseId(1L);
+        modelReq.setDatabaseId(s2Database.getId());
-        modelReq.setDomainId(3L);
         modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
         modelReq.setViewOrgs(Collections.singletonList("1"));
         modelReq.setAdmins(Collections.singletonList("admin"));
@@ -121,16 +126,16 @@ public class BenchMarkDemoDataLoader {
         modelDetail.setQueryType("sql_query");
         modelDetail.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre");
         modelReq.setModelDetail(modelDetail);
-        modelService.createModel(modelReq, user);
+        return modelService.createModel(modelReq, user);
     }
 
-    public void addModel_2() throws Exception {
+    public ModelResp addModel_2(DomainResp s2Domain, DatabaseResp s2Database) throws Exception {
         ModelReq modelReq = new ModelReq();
-        modelReq.setDomainId(3L);
+        modelReq.setDomainId(s2Domain.getId());
         modelReq.setName("艺术家");
         modelReq.setBizName("artist");
         modelReq.setDescription("艺术家");
-        modelReq.setDatabaseId(1L);
+        modelReq.setDatabaseId(s2Database.getId());
         ModelDetail modelDetail = new ModelDetail();
         List<Dim> dimensions = new ArrayList<>();
         dimensions.add(new Dim("艺术家名称", "artist_name", DimensionType.categorical.name(), 1));
@@ -148,16 +153,16 @@ public class BenchMarkDemoDataLoader {
         modelDetail.setQueryType("sql_query");
         modelDetail.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist");
         modelReq.setModelDetail(modelDetail);
-        modelService.createModel(modelReq, user);
+        return modelService.createModel(modelReq, user);
     }
 
-    public void addModel_3() throws Exception {
+    public ModelResp addModel_3(DomainResp s2Domain, DatabaseResp s2Database) throws Exception {
         ModelReq modelReq = new ModelReq();
-        modelReq.setDomainId(3L);
+        modelReq.setDomainId(s2Domain.getId());
         modelReq.setName("文件");
         modelReq.setBizName("files");
         modelReq.setDescription("文件");
-        modelReq.setDatabaseId(1L);
+        modelReq.setDatabaseId(s2Database.getId());
         ModelDetail modelDetail = new ModelDetail();
         List<Dim> dimensions = new ArrayList<>();
         dimensions.add(new Dim("持续时间", "duration", DimensionType.categorical.name(), 1));
@@ -175,16 +180,16 @@ public class BenchMarkDemoDataLoader {
         modelDetail.setQueryType("sql_query");
         modelDetail.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files");
         modelReq.setModelDetail(modelDetail);
-        modelService.createModel(modelReq, user);
+        return modelService.createModel(modelReq, user);
     }
 
-    public void addModel_4() throws Exception {
+    public ModelResp addModel_4(DomainResp s2Domain, DatabaseResp s2Database) throws Exception {
         ModelReq modelReq = new ModelReq();
-        modelReq.setDomainId(3L);
+        modelReq.setDomainId(s2Domain.getId());
         modelReq.setName("歌曲");
         modelReq.setBizName("song");
         modelReq.setDescription("歌曲");
-        modelReq.setDatabaseId(1L);
+        modelReq.setDatabaseId(s2Database.getId());
         ModelDetail modelDetail = new ModelDetail();
         List<Dim> dimensions = new ArrayList<>();
         Dim dimension1 = new Dim("", "imp_date", DimensionType.time.name(), 0);
@@ -213,22 +218,17 @@ public class BenchMarkDemoDataLoader {
         modelDetail.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, "
                 + " rating, languages, releasedate, resolution FROM song");
         modelReq.setModelDetail(modelDetail);
-        modelService.createModel(modelReq, user);
+        return modelService.createModel(modelReq, user);
     }
 
-    public void addDataSet_1() {
+    public void addDataSet_1(DomainResp s2Domain) {
         DataSetReq viewReq = new DataSetReq();
         viewReq.setName("cspider");
         viewReq.setBizName("singer");
-        viewReq.setDomainId(3L);
+        viewReq.setDomainId(s2Domain.getId());
         viewReq.setDescription("包含cspider数据集相关标签和指标信息");
         viewReq.setAdmins(Lists.newArrayList("admin"));
-        List<DataSetModelConfig> viewModelConfigs = Lists.newArrayList(
+        List<DataSetModelConfig> viewModelConfigs = modelDemoDataLoader.getDataSetModelConfigs(s2Domain.getId());
-                new DataSetModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()),
-                new DataSetModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()),
-                new DataSetModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()),
-                new DataSetModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L))
-        );
         DataSetDetail viewDetail = new DataSetDetail();
         viewDetail.setDataSetModelConfigs(viewModelConfigs);
         viewReq.setDataSetDetail(viewDetail);
@@ -254,61 +254,61 @@ public class BenchMarkDemoDataLoader {
         viewService.save(viewReq, User.getFakeUser());
     }
 
-    public void addModelRela_1() {
+    public void addModelRela_1(DomainResp s2Domain, ModelResp genreModelResp, ModelResp artistModelResp) {
         List<JoinCondition> joinConditions = Lists.newArrayList();
         joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
-        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setDomainId(s2Domain.getId());
-        modelRelaReq.setFromModelId(6L);
+        modelRelaReq.setFromModelId(artistModelResp.getId());
-        modelRelaReq.setToModelId(5L);
+        modelRelaReq.setToModelId(genreModelResp.getId());
         modelRelaReq.setJoinType("left join");
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);
     }
 
-    public void addModelRela_2() {
+    public void addModelRela_2(DomainResp s2Domain, ModelResp filesModelResp, ModelResp artistModelResp) {
         List<JoinCondition> joinConditions = Lists.newArrayList();
         joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
-        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setDomainId(s2Domain.getId());
-        modelRelaReq.setFromModelId(7L);
+        modelRelaReq.setFromModelId(filesModelResp.getId());
-        modelRelaReq.setToModelId(6L);
+        modelRelaReq.setToModelId(artistModelResp.getId());
         modelRelaReq.setJoinType("left join");
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);
     }
 
-    public void addModelRela_3() {
+    public void addModelRela_3(DomainResp s2Domain, ModelResp songModelResp, ModelResp artistModelResp) {
         List<JoinCondition> joinConditions = Lists.newArrayList();
         joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
-        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setDomainId(s2Domain.getId());
-        modelRelaReq.setFromModelId(8L);
+        modelRelaReq.setFromModelId(songModelResp.getId());
-        modelRelaReq.setToModelId(6L);
+        modelRelaReq.setToModelId(artistModelResp.getId());
         modelRelaReq.setJoinType("left join");
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);
     }
 
-    public void addModelRela_4() {
+    public void addModelRela_4(DomainResp s2Domain, ModelResp songModelResp, ModelResp genreModelResp) {
         List<JoinCondition> joinConditions = Lists.newArrayList();
         joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
-        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setDomainId(s2Domain.getId());
-        modelRelaReq.setFromModelId(8L);
+        modelRelaReq.setFromModelId(songModelResp.getId());
-        modelRelaReq.setToModelId(5L);
+        modelRelaReq.setToModelId(genreModelResp.getId());
         modelRelaReq.setJoinType("left join");
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);
     }
 
-    public void addModelRela_5() {
+    public void addModelRela_5(DomainResp s2Domain, ModelResp songModelResp, ModelResp filesModelResp) {
         List<JoinCondition> joinConditions = Lists.newArrayList();
         joinConditions.add(new JoinCondition("f_id", "f_id", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
-        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setDomainId(s2Domain.getId());
-        modelRelaReq.setFromModelId(8L);
+        modelRelaReq.setFromModelId(songModelResp.getId());
-        modelRelaReq.setToModelId(7L);
+        modelRelaReq.setToModelId(filesModelResp.getId());
         modelRelaReq.setJoinType("left join");
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);

launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java

@@ -91,6 +91,7 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ModelDemoDataLoader {
 
+    protected DatabaseResp tmpDatabaseResp = null;
     private User user = User.getFakeUser();
     @Autowired
     private DatabaseService databaseService;
@@ -122,6 +123,7 @@ public class ModelDemoDataLoader {
     public void doRun() {
         try {
             DatabaseResp databaseResp = addDatabase();
+            tmpDatabaseResp = databaseResp;
             DomainResp s2Domain = addDomain();
             TagObjectResp s2TagObject = addTagObjectUser(s2Domain);
             ModelResp userModel = addModel_1(s2Domain, databaseResp, s2TagObject);
@@ -537,7 +539,7 @@ public class ModelDemoDataLoader {
         DataSetReq dataSetReq = new DataSetReq();
         dataSetReq.setName("超音数");
         dataSetReq.setBizName("s2");
-        dataSetReq.setDomainId(1L);
+        dataSetReq.setDomainId(s2Domain.getId());
         dataSetReq.setDescription("包含超音数访问统计相关的指标和维度等");
         dataSetReq.setAdmins(Lists.newArrayList("admin"));
         List<DataSetModelConfig> dataSetModelConfigs = getDataSetModelConfigs(s2Domain.getId());
@@ -680,7 +682,7 @@ public class ModelDemoDataLoader {
         return metricService.getMetric(model.getId(), bizName);
     }
 
-    private List<DataSetModelConfig> getDataSetModelConfigs(Long domainId) {
+    protected List<DataSetModelConfig> getDataSetModelConfigs(Long domainId) {
         List<DataSetModelConfig> dataSetModelConfigs = Lists.newArrayList();
         List<ModelResp> modelByDomainIds =
                 modelService.getModelByDomainIds(Lists.newArrayList(domainId));

launchers/standalone/src/main/resources/application-local.yaml

@@ -114,4 +114,4 @@ logging:
 
 inMemoryEmbeddingStore:
   persistent:
-    path: /tmp
+    path: /tmp