From 6fcfdc15e13e5d38a9afcf0b7c44d745fd6cc40a Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Sun, 5 Jan 2025 15:42:54 +0800 Subject: [PATCH] [improvement][headless]Remove deprecated tag related constructs. --- .../headless/api/pojo/MetaFilter.java | 4 ++ .../api/pojo/response/DataSetResp.java | 4 -- .../api/pojo/response/SemanticSchemaResp.java | 1 - .../chat/parser/llm/LLMSqlParser.java | 3 +- .../service/impl/DataSetServiceImpl.java | 33 +---------- .../service/impl/SchemaServiceImpl.java | 56 +++---------------- .../headless/server/utils/DictUtils.java | 12 +--- 7 files changed, 17 insertions(+), 96 deletions(-) diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java index b71cca0e2..a5eb75e8e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetaFilter.java @@ -1,12 +1,16 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.base.Objects; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import java.util.List; @Data +@Builder +@AllArgsConstructor @NoArgsConstructor public class MetaFilter { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java index 82532f46d..e85ca0ea2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java @@ -30,10 +30,6 @@ public class DataSetResp extends SchemaItem { private QueryConfig queryConfig = new QueryConfig(); - private List allMetrics = new ArrayList<>(); - - private List allDimensions = new ArrayList<>(); - public List metricIds() { return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics) .flatMap(Collection::stream).collect(Collectors.toList()); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index 88b6aeedb..e456a4b05 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -22,7 +22,6 @@ public class SemanticSchemaResp { private SchemaType schemaType; private List metrics = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); - private List tags = Lists.newArrayList(); private List modelRelas = Lists.newArrayList(); private List modelResps = Lists.newArrayList(); private DataSetResp dataSetResp; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java index 8bcc95ac3..bf541ad94 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java @@ -35,7 +35,8 @@ public class LLMSqlParser implements SemanticParser { if (dataSetId == null) { return; } - log.info("try generating query statement for dataSetId:{}", dataSetId); + log.info("try generating query statement for query:{}, dataSetId:{}", + queryCtx.getRequest().getQueryText(), dataSetId); // 3.invoke LLM service to do parsing. tryParse(queryCtx, dataSetId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java index 8abff2ba5..3df28ce9f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java @@ -15,24 +15,14 @@ import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.DataSetDetail; import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.QueryConfig; -import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; -import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.TagItem; import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.DomainService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.TagMetaService; +import com.tencent.supersonic.headless.server.service.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -42,14 +32,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.Arrays; -import java.util.Comparator; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -69,10 +52,6 @@ public class DataSetServiceImpl extends ServiceImpl @Autowired private MetricService metricService; - @Lazy - @Autowired - private TagMetaService tagMetaService; - @Override public DataSetResp save(DataSetReq dataSetReq, User user) { dataSetReq.createdBy(user.getName()); @@ -199,13 +178,7 @@ public class DataSetServiceImpl extends ServiceImpl dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(","))); dataSetResp.setTypeEnum(TypeEnums.DATASET); - List dimensionItems = - tagMetaService.getTagItems(dataSetResp.dimensionIds(), TagDefineType.DIMENSION); - dataSetResp.setAllDimensions(dimensionItems); - List metricItems = - tagMetaService.getTagItems(dataSetResp.metricIds(), TagDefineType.METRIC); - dataSetResp.setAllMetrics(metricItems); return dataSetResp; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index b27c9de5f..d38bb1bef 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -11,30 +11,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.TagResp; -import com.tencent.supersonic.headless.api.pojo.response.TermResp; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.manager.DimensionYamlManager; import com.tencent.supersonic.headless.server.manager.MetricYamlManager; import com.tencent.supersonic.headless.server.manager.ModelYamlManager; @@ -43,16 +25,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.DatabaseService; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.DomainService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.ModelRelaService; -import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.SchemaService; -import com.tencent.supersonic.headless.server.service.TagMetaService; -import com.tencent.supersonic.headless.server.service.TermService; +import com.tencent.supersonic.headless.server.service.*; import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; @@ -64,14 +37,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -97,7 +63,6 @@ public class SchemaServiceImpl implements SchemaService { private final DomainService domainService; private final DataSetService dataSetService; private final ModelRelaService modelRelaService; - private final TagMetaService tagService; private final TermService termService; private final DatabaseService databaseService; @@ -106,8 +71,8 @@ public class SchemaServiceImpl implements SchemaService { public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService, MetricService metricService, DomainService domainService, DataSetService dataSetService, - ModelRelaService modelRelaService, StatUtils statUtils, TagMetaService tagService, - TermService termService, DatabaseService databaseService) { + ModelRelaService modelRelaService, StatUtils statUtils, TermService termService, + DatabaseService databaseService) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; @@ -115,7 +80,6 @@ public class SchemaServiceImpl implements SchemaService { this.dataSetService = dataSetService; this.modelRelaService = modelRelaService; this.statUtils = statUtils; - this.tagService = tagService; this.termService = termService; this.databaseService = databaseService; } @@ -408,13 +372,7 @@ public class SchemaServiceImpl implements SchemaService { modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); semanticSchemaResp.setSchemaType(SchemaType.MODEL); } - if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { - // add tag info - TagFilter tagFilter = new TagFilter(); - tagFilter.setModelIds(semanticSchemaResp.getModelIds()); - List tagResps = tagService.getTags(tagFilter); - semanticSchemaResp.setTags(tagResps); - } + if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index c906ad098..3dc313cc3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -17,13 +17,11 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.TagMetaService; import com.xkzhangsan.time.utils.CollectionUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -60,16 +58,13 @@ public class DictUtils { private final MetricService metricService; private final SemanticLayerService queryService; private final ModelService modelService; - private final TagMetaService tagMetaService; public DictUtils(DimensionService dimensionService, MetricService metricService, - SemanticLayerService queryService, ModelService modelService, - @Lazy TagMetaService tagMetaService) { + SemanticLayerService queryService, ModelService modelService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; this.modelService = modelService; - this.tagMetaService = tagMetaService; } public String fetchDictFileName(DictItemResp dictItemResp) { @@ -133,11 +128,6 @@ public class DictUtils { dictItemResp.setModelId(dimension.getModelId()); dictItemResp.setBizName(dimension.getBizName()); } - if (TypeEnums.TAG.equals(TypeEnums.valueOf(dictConfDO.getType()))) { - TagResp tagResp = tagMetaService.getTag(dictConfDO.getItemId(), User.getDefaultUser()); - dictItemResp.setModelId(tagResp.getModelId()); - dictItemResp.setBizName(tagResp.getBizName()); - } return dictItemResp; }