diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java index cc2a89ea6..10054f423 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java @@ -17,7 +17,6 @@ import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.util.ComponentFactory; import com.tencent.supersonic.chat.server.util.QueryReqConverter; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; @@ -45,6 +44,8 @@ public class ChatServiceImpl implements ChatService { private ChatQueryService chatQueryService; @Autowired private RetrieveService retrieveService; + @Autowired + private AgentService agentService; private List chatParsers = ComponentFactory.getChatParsers(); private List chatExecutors = ComponentFactory.getChatExecutors(); private List parseResultProcessors = ComponentFactory.getParseProcessors(); @@ -111,7 +112,6 @@ public class ChatServiceImpl implements ChatService { private ChatParseContext buildParseContext(ChatParseReq chatParseReq) { ChatParseContext chatParseContext = new ChatParseContext(); BeanMapper.mapper(chatParseReq, chatParseContext); - AgentService agentService = ContextUtils.getBean(AgentService.class); Agent agent = agentService.getAgent(chatParseReq.getAgentId()); chatParseContext.setAgent(agent); QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext); @@ -147,6 +147,9 @@ public class ChatServiceImpl implements ChatService { @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { + Integer agentId = dimensionValueReq.getAgentId(); + Agent agent = agentService.getAgent(agentId); + dimensionValueReq.setDataSetIds(agent.getDataSetIds()); return chatQueryService.queryDimensionValue(dimensionValueReq, user); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java index 82b361d1c..864e30266 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java @@ -1,7 +1,8 @@ package com.tencent.supersonic.headless.api.pojo.request; -import javax.validation.constraints.NotNull; import lombok.Data; +import javax.validation.constraints.NotNull; +import java.util.Set; @Data public class DimensionValueReq { @@ -17,4 +18,7 @@ public class DimensionValueReq { @NotNull private String value; + + private Set dataSetIds; + } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java index cd269d94c..ffaf657bf 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java @@ -189,7 +189,7 @@ public class SearchService { String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT + dimensionValueReq.getElementID(); PriorityQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); - if (org.apache.commons.collections.CollectionUtils.isEmpty(terms)) { + if (CollectionUtils.isEmpty(terms)) { return new ArrayList<>(); } return terms.stream().map(term -> term.getWord()).collect(Collectors.toList()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java index d708b61d3..860b363dc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java @@ -2,7 +2,11 @@ package com.tencent.supersonic.headless.server.facade.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.jsqlparser.FieldExpression; +import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; +import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; @@ -10,11 +14,6 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.common.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; -import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; -import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.EntityInfo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; @@ -32,15 +31,16 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo; import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; import com.tencent.supersonic.headless.api.pojo.response.MapResp; @@ -48,6 +48,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.chat.ChatContext; +import com.tencent.supersonic.headless.chat.QueryContext; import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector; import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector; import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult; @@ -59,29 +61,27 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; import com.tencent.supersonic.headless.chat.query.QueryManager; import com.tencent.supersonic.headless.chat.query.SemanticQuery; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery; -import com.tencent.supersonic.headless.chat.ChatContext; -import com.tencent.supersonic.headless.chat.QueryContext; +import com.tencent.supersonic.headless.server.facade.service.ChatQueryService; +import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.utils.ComponentFactory; import com.tencent.supersonic.headless.server.web.service.ChatContextService; -import com.tencent.supersonic.headless.server.facade.service.ChatQueryService; import com.tencent.supersonic.headless.server.web.service.DataSetService; -import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.web.service.SchemaService; import com.tencent.supersonic.headless.server.web.service.WorkflowService; -import com.tencent.supersonic.headless.server.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; import net.sf.jsqlparser.expression.operators.relational.EqualsTo; -import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; import net.sf.jsqlparser.expression.operators.relational.GreaterThan; import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; import net.sf.jsqlparser.expression.operators.relational.InExpression; import net.sf.jsqlparser.expression.operators.relational.MinorThan; import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; +import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; import net.sf.jsqlparser.schema.Column; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -539,12 +539,10 @@ public class ChatQueryServiceImpl implements ChatQueryService { @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { SemanticQueryResp semanticQueryResp = new SemanticQueryResp(); - SemanticSchema semanticSchema = schemaService.getSemanticSchema(); - SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID()); - Set detectDataSetIds = new HashSet<>(); - detectDataSetIds.add(schemaElement.getDataSet()); - dimensionValueReq.setModelId(schemaElement.getModel()); - List dimensionValues = getDimensionValues(dimensionValueReq, detectDataSetIds); + DimensionResp dimensionResp = schemaService.getDimension(dimensionValueReq.getElementID()); + Set dataSetIds = dimensionValueReq.getDataSetIds(); + dimensionValueReq.setModelId(dimensionResp.getModelId()); + List dimensionValues = getDimensionValues(dimensionValueReq, dataSetIds); // if the search results is null,search dimensionValue from database if (CollectionUtils.isEmpty(dimensionValues)) { semanticQueryResp = queryDatabase(dimensionValueReq, user); @@ -593,21 +591,12 @@ public class ChatQueryServiceImpl implements ChatQueryService { .collect(Collectors.toList()); } - private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) throws Exception { - QueryStructReq queryStructReq = new QueryStructReq(); - - DateConf dateConf = new DateConf(); - dateConf.setDateMode(DateConf.DateMode.RECENT); - dateConf.setUnit(1); - dateConf.setPeriod("DAY"); - queryStructReq.setDateInfo(dateConf); - queryStructReq.setLimit(20L); - queryStructReq.setDataSetId(dimensionValueReq.getModelId()); - queryStructReq.setQueryType(QueryType.ID); - List groups = new ArrayList<>(); - groups.add(dimensionValueReq.getBizName()); - queryStructReq.setGroups(groups); - return semanticLayerService.queryByReq(queryStructReq, user); + private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) { + QueryDimValueReq queryDimValueReq = new QueryDimValueReq(); + queryDimValueReq.setValue(dimensionValueReq.getValue()); + queryDimValueReq.setModelId(dimensionValueReq.getModelId()); + queryDimValueReq.setDimensionBizName(dimensionValueReq.getBizName()); + return semanticLayerService.queryDimValue(queryDimValueReq, user); } public void correct(QuerySqlReq querySqlReq, User user) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index 7ea69c14e..196f40b74 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -48,6 +48,13 @@ import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.web.service.DataSetService; import com.tencent.supersonic.headless.server.web.service.SchemaService; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; + import java.time.LocalDate; import java.util.ArrayList; import java.util.LinkedHashSet; @@ -56,12 +63,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Service; @Service @@ -246,13 +247,17 @@ public class S2SemanticLayerService implements SemanticLayerService { DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getDimensionBizName(), queryDimValueReq.getModelId()); ModelResp modelResp = modelResps.get(0); - String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName()); + String sql = String.format("select distinct %s from %s where 1=1", + dimensionResp.getName(), modelResp.getName()); List timeDims = modelResp.getTimeDimension(); if (CollectionUtils.isNotEmpty(timeDims)) { - sql = String.format("%s where %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(), + sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate()); } + if (StringUtils.isNotBlank(queryDimValueReq.getValue())) { + sql += " AND " + queryDimValueReq.getDimensionBizName() + " LIKE '%" + queryDimValueReq.getValue() + "%'"; + } querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); querySqlReq.setSql(sql); return querySqlReq; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java index e19c81317..2809b0c34 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java @@ -40,6 +40,8 @@ public interface SchemaService { DimensionResp getDimension(String bizName, Long modelId); + DimensionResp getDimension(Long id); + List getMetrics(MetaFilter metaFilter); List getDomainList(User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java index 445230393..7645c1e2a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java @@ -317,6 +317,11 @@ public class SchemaServiceImpl implements SchemaService { return dimensionService.getDimension(bizName, modelId); } + @Override + public DimensionResp getDimension(Long id) { + return dimensionService.getDimension(id); + } + @Override public List getDimensions(MetaFilter metaFilter) { return dimensionService.getDimensions(metaFilter);