From 418abef9821f894d50d56677d20f556010208209 Mon Sep 17 00:00:00 2001 From: mainmain <57514971+mainmainer@users.noreply.github.com> Date: Wed, 22 May 2024 15:56:06 +0800 Subject: [PATCH] (improvement)(Headless) corrector supports subselect sql and opt BenchMarkDemoDataLoader (#1023) * [improvement] corrector and QueryReqConverter support subselect * [improvement] corrector and QueryReqConverter support subselect * [improvement] adapt BenchMarkDemoDataLoader * [improvement] adapt checkstyle --------- Co-authored-by: zuopengge --- .../common/util/jsqlparser/SqlAddHelper.java | 21 ++++ .../util/jsqlparser/SqlReplaceHelper.java | 10 ++ .../supersonic/BenchMarkDemoDataLoader.java | 118 +++++++++--------- .../supersonic/ModelDemoDataLoader.java | 6 +- .../src/main/resources/application-local.yaml | 2 +- 5 files changed, 95 insertions(+), 62 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java index 835ddf884..2ea2137e7 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java @@ -5,6 +5,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; + +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; @@ -23,6 +25,7 @@ import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.statement.select.SelectItem; import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.ParenthesedSelect; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -148,6 +151,24 @@ public class SqlAddHelper { return sql; } PlainSelect plainSelect = (PlainSelect) selectStatement; + List chNameList = TimeDimensionEnum.getChNameList(); + Boolean dateWhere = false; + for (String chName : chNameList) { + if (expression.toString().contains(chName)) { + dateWhere = true; + } + } + if (plainSelect.getFromItem() instanceof ParenthesedSelect && dateWhere) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); + PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); + Expression subWhere = subPlainSelect.getWhere(); + if (subWhere == null) { + subPlainSelect.setWhere(expression); + } else { + subPlainSelect.setWhere(new AndExpression(subWhere, expression)); + } + return selectStatement.toString(); + } Expression where = plainSelect.getWhere(); if (where == null) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java index 321fbb065..0f1a4aa3e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java @@ -397,12 +397,14 @@ public class SqlReplaceHelper { if (selectStatement instanceof PlainSelect) { PlainSelect plainSelect = (PlainSelect) selectStatement; replaceSingleTable(plainSelect, tableName); + replaceSubTable(plainSelect, tableName); } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { setOperationList.getSelects().forEach(subSelectBody -> { PlainSelect subPlainSelect = (PlainSelect) subSelectBody; replaceSingleTable(subPlainSelect, tableName); + replaceSubTable(subPlainSelect, tableName); }); } } @@ -410,6 +412,14 @@ public class SqlReplaceHelper { return selectStatement.toString(); } + public static void replaceSubTable(PlainSelect plainSelect, String tableName) { + if (plainSelect.getFromItem() instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); + PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); + replaceSingleTable(subPlainSelect, tableName); + } + } + public static void replaceSingleTable(PlainSelect plainSelect, String tableName) { // replace table name List plainSelects = new ArrayList<>(); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java index 90ae79e08..735fbd09a 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java @@ -25,6 +25,9 @@ import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.DomainResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; @@ -55,28 +58,31 @@ public class BenchMarkDemoDataLoader { private DataSetService viewService; @Autowired private MetricService metricService; + @Autowired + private ModelDemoDataLoader modelDemoDataLoader; public void doRun() { try { - addDomain(); - addModel_1(); - addModel_2(); - addModel_3(); - addModel_4(); - addDataSet_1(); - addModelRela_1(); - addModelRela_2(); - addModelRela_3(); - addModelRela_4(); - addModelRela_5(); - batchPushlishMetric(); + DatabaseResp databaseResp = modelDemoDataLoader.tmpDatabaseResp; + DomainResp s2Domain = addDomain(); + ModelResp genreModelResp = addModel_1(s2Domain, databaseResp); + ModelResp artistModelResp = addModel_2(s2Domain, databaseResp); + ModelResp filesModelResp = addModel_3(s2Domain, databaseResp); + ModelResp songModelResp = addModel_4(s2Domain, databaseResp); + addDataSet_1(s2Domain); + addModelRela_1(s2Domain, genreModelResp, artistModelResp); + addModelRela_2(s2Domain, filesModelResp, artistModelResp); + addModelRela_3(s2Domain, songModelResp, artistModelResp); + addModelRela_4(s2Domain, songModelResp, genreModelResp); + addModelRela_5(s2Domain, songModelResp, filesModelResp); + //batchPushlishMetric(); } catch (Exception e) { log.error("Failed to add bench mark demo data", e); } } - public void addDomain() { + public DomainResp addDomain() { DomainReq domainReq = new DomainReq(); domainReq.setName("测评数据_音乐"); domainReq.setBizName("music"); @@ -85,17 +91,16 @@ public class BenchMarkDemoDataLoader { domainReq.setViewOrgs(Collections.singletonList("1")); domainReq.setAdmins(Collections.singletonList("admin")); domainReq.setAdminOrgs(Collections.emptyList()); - domainService.createDomain(domainReq, user); + return domainService.createDomain(domainReq, user); } - public void addModel_1() throws Exception { + public ModelResp addModel_1(DomainResp s2Domain, DatabaseResp s2Database) throws Exception { ModelReq modelReq = new ModelReq(); - modelReq.setDomainId(3L); + modelReq.setDomainId(s2Domain.getId()); modelReq.setName("艺术类型"); modelReq.setBizName("genre"); modelReq.setDescription("艺术类型"); - modelReq.setDatabaseId(1L); - modelReq.setDomainId(3L); + modelReq.setDatabaseId(s2Database.getId()); modelReq.setViewers(Arrays.asList("admin", "tom", "jack")); modelReq.setViewOrgs(Collections.singletonList("1")); modelReq.setAdmins(Collections.singletonList("admin")); @@ -121,16 +126,16 @@ public class BenchMarkDemoDataLoader { modelDetail.setQueryType("sql_query"); modelDetail.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre"); modelReq.setModelDetail(modelDetail); - modelService.createModel(modelReq, user); + return modelService.createModel(modelReq, user); } - public void addModel_2() throws Exception { + public ModelResp addModel_2(DomainResp s2Domain, DatabaseResp s2Database) throws Exception { ModelReq modelReq = new ModelReq(); - modelReq.setDomainId(3L); + modelReq.setDomainId(s2Domain.getId()); modelReq.setName("艺术家"); modelReq.setBizName("artist"); modelReq.setDescription("艺术家"); - modelReq.setDatabaseId(1L); + modelReq.setDatabaseId(s2Database.getId()); ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); dimensions.add(new Dim("艺术家名称", "artist_name", DimensionType.categorical.name(), 1)); @@ -148,16 +153,16 @@ public class BenchMarkDemoDataLoader { modelDetail.setQueryType("sql_query"); modelDetail.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist"); modelReq.setModelDetail(modelDetail); - modelService.createModel(modelReq, user); + return modelService.createModel(modelReq, user); } - public void addModel_3() throws Exception { + public ModelResp addModel_3(DomainResp s2Domain, DatabaseResp s2Database) throws Exception { ModelReq modelReq = new ModelReq(); - modelReq.setDomainId(3L); + modelReq.setDomainId(s2Domain.getId()); modelReq.setName("文件"); modelReq.setBizName("files"); modelReq.setDescription("文件"); - modelReq.setDatabaseId(1L); + modelReq.setDatabaseId(s2Database.getId()); ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); dimensions.add(new Dim("持续时间", "duration", DimensionType.categorical.name(), 1)); @@ -175,16 +180,16 @@ public class BenchMarkDemoDataLoader { modelDetail.setQueryType("sql_query"); modelDetail.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files"); modelReq.setModelDetail(modelDetail); - modelService.createModel(modelReq, user); + return modelService.createModel(modelReq, user); } - public void addModel_4() throws Exception { + public ModelResp addModel_4(DomainResp s2Domain, DatabaseResp s2Database) throws Exception { ModelReq modelReq = new ModelReq(); - modelReq.setDomainId(3L); + modelReq.setDomainId(s2Domain.getId()); modelReq.setName("歌曲"); modelReq.setBizName("song"); modelReq.setDescription("歌曲"); - modelReq.setDatabaseId(1L); + modelReq.setDatabaseId(s2Database.getId()); ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionType.time.name(), 0); @@ -213,22 +218,17 @@ public class BenchMarkDemoDataLoader { modelDetail.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, " + " rating, languages, releasedate, resolution FROM song"); modelReq.setModelDetail(modelDetail); - modelService.createModel(modelReq, user); + return modelService.createModel(modelReq, user); } - public void addDataSet_1() { + public void addDataSet_1(DomainResp s2Domain) { DataSetReq viewReq = new DataSetReq(); viewReq.setName("cspider"); viewReq.setBizName("singer"); - viewReq.setDomainId(3L); + viewReq.setDomainId(s2Domain.getId()); viewReq.setDescription("包含cspider数据集相关标签和指标信息"); viewReq.setAdmins(Lists.newArrayList("admin")); - List viewModelConfigs = Lists.newArrayList( - new DataSetModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()), - new DataSetModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()), - new DataSetModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()), - new DataSetModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L)) - ); + List viewModelConfigs = modelDemoDataLoader.getDataSetModelConfigs(s2Domain.getId()); DataSetDetail viewDetail = new DataSetDetail(); viewDetail.setDataSetModelConfigs(viewModelConfigs); viewReq.setDataSetDetail(viewDetail); @@ -254,61 +254,61 @@ public class BenchMarkDemoDataLoader { viewService.save(viewReq, User.getFakeUser()); } - public void addModelRela_1() { + public void addModelRela_1(DomainResp s2Domain, ModelResp genreModelResp, ModelResp artistModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); - modelRelaReq.setDomainId(3L); - modelRelaReq.setFromModelId(6L); - modelRelaReq.setToModelId(5L); + modelRelaReq.setDomainId(s2Domain.getId()); + modelRelaReq.setFromModelId(artistModelResp.getId()); + modelRelaReq.setToModelId(genreModelResp.getId()); modelRelaReq.setJoinType("left join"); modelRelaReq.setJoinConditions(joinConditions); modelRelaService.save(modelRelaReq, user); } - public void addModelRela_2() { + public void addModelRela_2(DomainResp s2Domain, ModelResp filesModelResp, ModelResp artistModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); - modelRelaReq.setDomainId(3L); - modelRelaReq.setFromModelId(7L); - modelRelaReq.setToModelId(6L); + modelRelaReq.setDomainId(s2Domain.getId()); + modelRelaReq.setFromModelId(filesModelResp.getId()); + modelRelaReq.setToModelId(artistModelResp.getId()); modelRelaReq.setJoinType("left join"); modelRelaReq.setJoinConditions(joinConditions); modelRelaService.save(modelRelaReq, user); } - public void addModelRela_3() { + public void addModelRela_3(DomainResp s2Domain, ModelResp songModelResp, ModelResp artistModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); - modelRelaReq.setDomainId(3L); - modelRelaReq.setFromModelId(8L); - modelRelaReq.setToModelId(6L); + modelRelaReq.setDomainId(s2Domain.getId()); + modelRelaReq.setFromModelId(songModelResp.getId()); + modelRelaReq.setToModelId(artistModelResp.getId()); modelRelaReq.setJoinType("left join"); modelRelaReq.setJoinConditions(joinConditions); modelRelaService.save(modelRelaReq, user); } - public void addModelRela_4() { + public void addModelRela_4(DomainResp s2Domain, ModelResp songModelResp, ModelResp genreModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); - modelRelaReq.setDomainId(3L); - modelRelaReq.setFromModelId(8L); - modelRelaReq.setToModelId(5L); + modelRelaReq.setDomainId(s2Domain.getId()); + modelRelaReq.setFromModelId(songModelResp.getId()); + modelRelaReq.setToModelId(genreModelResp.getId()); modelRelaReq.setJoinType("left join"); modelRelaReq.setJoinConditions(joinConditions); modelRelaService.save(modelRelaReq, user); } - public void addModelRela_5() { + public void addModelRela_5(DomainResp s2Domain, ModelResp songModelResp, ModelResp filesModelResp) { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("f_id", "f_id", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); - modelRelaReq.setDomainId(3L); - modelRelaReq.setFromModelId(8L); - modelRelaReq.setToModelId(7L); + modelRelaReq.setDomainId(s2Domain.getId()); + modelRelaReq.setFromModelId(songModelResp.getId()); + modelRelaReq.setToModelId(filesModelResp.getId()); modelRelaReq.setJoinType("left join"); modelRelaReq.setJoinConditions(joinConditions); modelRelaService.save(modelRelaReq, user); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index 08715c2a8..4899d05be 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -91,6 +91,7 @@ import java.util.stream.Collectors; @Slf4j public class ModelDemoDataLoader { + protected DatabaseResp tmpDatabaseResp = null; private User user = User.getFakeUser(); @Autowired private DatabaseService databaseService; @@ -122,6 +123,7 @@ public class ModelDemoDataLoader { public void doRun() { try { DatabaseResp databaseResp = addDatabase(); + tmpDatabaseResp = databaseResp; DomainResp s2Domain = addDomain(); TagObjectResp s2TagObject = addTagObjectUser(s2Domain); ModelResp userModel = addModel_1(s2Domain, databaseResp, s2TagObject); @@ -537,7 +539,7 @@ public class ModelDemoDataLoader { DataSetReq dataSetReq = new DataSetReq(); dataSetReq.setName("超音数"); dataSetReq.setBizName("s2"); - dataSetReq.setDomainId(1L); + dataSetReq.setDomainId(s2Domain.getId()); dataSetReq.setDescription("包含超音数访问统计相关的指标和维度等"); dataSetReq.setAdmins(Lists.newArrayList("admin")); List dataSetModelConfigs = getDataSetModelConfigs(s2Domain.getId()); @@ -680,7 +682,7 @@ public class ModelDemoDataLoader { return metricService.getMetric(model.getId(), bizName); } - private List getDataSetModelConfigs(Long domainId) { + protected List getDataSetModelConfigs(Long domainId) { List dataSetModelConfigs = Lists.newArrayList(); List modelByDomainIds = modelService.getModelByDomainIds(Lists.newArrayList(domainId)); diff --git a/launchers/standalone/src/main/resources/application-local.yaml b/launchers/standalone/src/main/resources/application-local.yaml index d755912b1..098dbfe37 100644 --- a/launchers/standalone/src/main/resources/application-local.yaml +++ b/launchers/standalone/src/main/resources/application-local.yaml @@ -114,4 +114,4 @@ logging: inMemoryEmbeddingStore: persistent: - path: /tmp \ No newline at end of file + path: /tmp