[improvement][headless]Remove deprecated tag related constructs.

This commit is contained in:
jerryjzhang
2025-01-05 15:42:54 +08:00
parent 4738b9d01b
commit 6fcfdc15e1
7 changed files with 17 additions and 96 deletions

View File

@@ -1,12 +1,16 @@
package com.tencent.supersonic.headless.api.pojo; package com.tencent.supersonic.headless.api.pojo;
import com.google.common.base.Objects; import com.google.common.base.Objects;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.List; import java.util.List;
@Data @Data
@Builder
@AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
public class MetaFilter { public class MetaFilter {

View File

@@ -30,10 +30,6 @@ public class DataSetResp extends SchemaItem {
private QueryConfig queryConfig = new QueryConfig(); private QueryConfig queryConfig = new QueryConfig();
private List<TagItem> allMetrics = new ArrayList<>();
private List<TagItem> allDimensions = new ArrayList<>();
public List<Long> metricIds() { public List<Long> metricIds() {
return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics) return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics)
.flatMap(Collection::stream).collect(Collectors.toList()); .flatMap(Collection::stream).collect(Collectors.toList());

View File

@@ -22,7 +22,6 @@ public class SemanticSchemaResp {
private SchemaType schemaType; private SchemaType schemaType;
private List<MetricSchemaResp> metrics = Lists.newArrayList(); private List<MetricSchemaResp> metrics = Lists.newArrayList();
private List<DimSchemaResp> dimensions = Lists.newArrayList(); private List<DimSchemaResp> dimensions = Lists.newArrayList();
private List<TagResp> tags = Lists.newArrayList();
private List<ModelRela> modelRelas = Lists.newArrayList(); private List<ModelRela> modelRelas = Lists.newArrayList();
private List<ModelResp> modelResps = Lists.newArrayList(); private List<ModelResp> modelResps = Lists.newArrayList();
private DataSetResp dataSetResp; private DataSetResp dataSetResp;

View File

@@ -35,7 +35,8 @@ public class LLMSqlParser implements SemanticParser {
if (dataSetId == null) { if (dataSetId == null) {
return; return;
} }
log.info("try generating query statement for dataSetId:{}", dataSetId); log.info("try generating query statement for query:{}, dataSetId:{}",
queryCtx.getRequest().getQueryText(), dataSetId);
// 3.invoke LLM service to do parsing. // 3.invoke LLM service to do parsing.
tryParse(queryCtx, dataSetId); tryParse(queryCtx, dataSetId);

View File

@@ -15,24 +15,14 @@ import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail; import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; import com.tencent.supersonic.headless.api.pojo.request.*;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.TagItem;
import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper; import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper;
import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.*;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
@@ -42,14 +32,7 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.Arrays; import java.util.*;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -69,10 +52,6 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
@Autowired @Autowired
private MetricService metricService; private MetricService metricService;
@Lazy
@Autowired
private TagMetaService tagMetaService;
@Override @Override
public DataSetResp save(DataSetReq dataSetReq, User user) { public DataSetResp save(DataSetReq dataSetReq, User user) {
dataSetReq.createdBy(user.getName()); dataSetReq.createdBy(user.getName());
@@ -199,13 +178,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList() dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList()
: Arrays.asList(dataSetDO.getAdminOrg().split(","))); : Arrays.asList(dataSetDO.getAdminOrg().split(",")));
dataSetResp.setTypeEnum(TypeEnums.DATASET); dataSetResp.setTypeEnum(TypeEnums.DATASET);
List<TagItem> dimensionItems =
tagMetaService.getTagItems(dataSetResp.dimensionIds(), TagDefineType.DIMENSION);
dataSetResp.setAllDimensions(dimensionItems);
List<TagItem> metricItems =
tagMetaService.getTagItems(dataSetResp.metricIds(), TagDefineType.METRIC);
dataSetResp.setAllMetrics(metricItems);
return dataSetResp; return dataSetResp;
} }

View File

@@ -11,30 +11,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; import com.tencent.supersonic.headless.api.pojo.enums.SchemaType;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager; import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager; import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager; import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
@@ -43,16 +25,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.*;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TermService;
import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder; import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder;
import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -64,14 +37,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -97,7 +63,6 @@ public class SchemaServiceImpl implements SchemaService {
private final DomainService domainService; private final DomainService domainService;
private final DataSetService dataSetService; private final DataSetService dataSetService;
private final ModelRelaService modelRelaService; private final ModelRelaService modelRelaService;
private final TagMetaService tagService;
private final TermService termService; private final TermService termService;
private final DatabaseService databaseService; private final DatabaseService databaseService;
@@ -106,8 +71,8 @@ public class SchemaServiceImpl implements SchemaService {
public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService, public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService,
MetricService metricService, DomainService domainService, DataSetService dataSetService, MetricService metricService, DomainService domainService, DataSetService dataSetService,
ModelRelaService modelRelaService, StatUtils statUtils, TagMetaService tagService, ModelRelaService modelRelaService, StatUtils statUtils, TermService termService,
TermService termService, DatabaseService databaseService) { DatabaseService databaseService) {
this.modelService = modelService; this.modelService = modelService;
this.dimensionService = dimensionService; this.dimensionService = dimensionService;
this.metricService = metricService; this.metricService = metricService;
@@ -115,7 +80,6 @@ public class SchemaServiceImpl implements SchemaService {
this.dataSetService = dataSetService; this.dataSetService = dataSetService;
this.modelRelaService = modelRelaService; this.modelRelaService = modelRelaService;
this.statUtils = statUtils; this.statUtils = statUtils;
this.tagService = tagService;
this.termService = termService; this.termService = termService;
this.databaseService = databaseService; this.databaseService = databaseService;
} }
@@ -408,13 +372,7 @@ public class SchemaServiceImpl implements SchemaService {
modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
semanticSchemaResp.setSchemaType(SchemaType.MODEL); semanticSchemaResp.setSchemaType(SchemaType.MODEL);
} }
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
// add tag info
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(semanticSchemaResp.getModelIds());
List<TagResp> tagResps = tagService.getTags(tagFilter);
semanticSchemaResp.setTags(tagResps);
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
DatabaseResp databaseResp = DatabaseResp databaseResp =
modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));

View File

@@ -17,13 +17,11 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.xkzhangsan.time.utils.CollectionUtil; import com.xkzhangsan.time.utils.CollectionUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -60,16 +58,13 @@ public class DictUtils {
private final MetricService metricService; private final MetricService metricService;
private final SemanticLayerService queryService; private final SemanticLayerService queryService;
private final ModelService modelService; private final ModelService modelService;
private final TagMetaService tagMetaService;
public DictUtils(DimensionService dimensionService, MetricService metricService, public DictUtils(DimensionService dimensionService, MetricService metricService,
SemanticLayerService queryService, ModelService modelService, SemanticLayerService queryService, ModelService modelService) {
@Lazy TagMetaService tagMetaService) {
this.dimensionService = dimensionService; this.dimensionService = dimensionService;
this.metricService = metricService; this.metricService = metricService;
this.queryService = queryService; this.queryService = queryService;
this.modelService = modelService; this.modelService = modelService;
this.tagMetaService = tagMetaService;
} }
public String fetchDictFileName(DictItemResp dictItemResp) { public String fetchDictFileName(DictItemResp dictItemResp) {
@@ -133,11 +128,6 @@ public class DictUtils {
dictItemResp.setModelId(dimension.getModelId()); dictItemResp.setModelId(dimension.getModelId());
dictItemResp.setBizName(dimension.getBizName()); dictItemResp.setBizName(dimension.getBizName());
} }
if (TypeEnums.TAG.equals(TypeEnums.valueOf(dictConfDO.getType()))) {
TagResp tagResp = tagMetaService.getTag(dictConfDO.getItemId(), User.getDefaultUser());
dictItemResp.setModelId(tagResp.getModelId());
dictItemResp.setBizName(tagResp.getBizName());
}
return dictItemResp; return dictItemResp;
} }