(fix)(Chat) Fix dimension value query failed (#1257)

Co-authored-by: lxwcodemonkey

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java

@@ -17,7 +17,6 @@ import com.tencent.supersonic.chat.server.service.ChatService;
 import com.tencent.supersonic.chat.server.util.ComponentFactory;
 import com.tencent.supersonic.chat.server.util.QueryReqConverter;
 import com.tencent.supersonic.common.util.BeanMapper;
-import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
@@ -45,6 +44,8 @@ public class ChatServiceImpl implements ChatService {
     private ChatQueryService chatQueryService;
     @Autowired
     private RetrieveService retrieveService;
+    @Autowired
+    private AgentService agentService;
     private List<ChatParser> chatParsers = ComponentFactory.getChatParsers();
     private List<ChatExecutor> chatExecutors = ComponentFactory.getChatExecutors();
     private List<ParseResultProcessor> parseResultProcessors = ComponentFactory.getParseProcessors();
@@ -111,7 +112,6 @@ public class ChatServiceImpl implements ChatService {
     private ChatParseContext buildParseContext(ChatParseReq chatParseReq) {
         ChatParseContext chatParseContext = new ChatParseContext();
         BeanMapper.mapper(chatParseReq, chatParseContext);
-        AgentService agentService = ContextUtils.getBean(AgentService.class);
         Agent agent = agentService.getAgent(chatParseReq.getAgentId());
         chatParseContext.setAgent(agent);
         QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
@@ -147,6 +147,9 @@ public class ChatServiceImpl implements ChatService {
 
     @Override
     public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
+        Integer agentId = dimensionValueReq.getAgentId();
+        Agent agent = agentService.getAgent(agentId);
+        dimensionValueReq.setDataSetIds(agent.getDataSetIds());
         return chatQueryService.queryDimensionValue(dimensionValueReq, user);
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java

@@ -1,7 +1,8 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
-import javax.validation.constraints.NotNull;
 import lombok.Data;
+import javax.validation.constraints.NotNull;
+import java.util.Set;
 
 @Data
 public class DimensionValueReq {
@@ -17,4 +18,7 @@ public class DimensionValueReq {
 
     @NotNull
     private String value;
+
+    private Set<Long> dataSetIds;
+
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java

@@ -189,7 +189,7 @@ public class SearchService {
         String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT
                 + dimensionValueReq.getElementID();
         PriorityQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
-        if (org.apache.commons.collections.CollectionUtils.isEmpty(terms)) {
+        if (CollectionUtils.isEmpty(terms)) {
             return new ArrayList<>();
         }
         return terms.stream().map(term -> term.getWord()).collect(Collectors.toList());

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java

@@ -2,7 +2,11 @@ package com.tencent.supersonic.headless.server.facade.service.impl;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.jsqlparser.FieldExpression;
+import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
@@ -10,11 +14,6 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.common.jsqlparser.FieldExpression;
-import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
 import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
 import com.tencent.supersonic.headless.api.pojo.EntityInfo;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
@@ -32,15 +31,16 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
+import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapResp;
@@ -48,6 +48,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
 import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
+import com.tencent.supersonic.headless.chat.ChatContext;
+import com.tencent.supersonic.headless.chat.QueryContext;
 import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector;
 import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector;
 import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
@@ -59,29 +61,27 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.chat.query.QueryManager;
 import com.tencent.supersonic.headless.chat.query.SemanticQuery;
 import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery;
-import com.tencent.supersonic.headless.chat.ChatContext;
+import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
-import com.tencent.supersonic.headless.chat.QueryContext;
+import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
 import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
+import com.tencent.supersonic.headless.server.utils.ComponentFactory;
 import com.tencent.supersonic.headless.server.web.service.ChatContextService;
-import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
 import com.tencent.supersonic.headless.server.web.service.DataSetService;
-import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
 import com.tencent.supersonic.headless.server.web.service.SchemaService;
 import com.tencent.supersonic.headless.server.web.service.WorkflowService;
-import com.tencent.supersonic.headless.server.utils.ComponentFactory;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.LongValue;
 import net.sf.jsqlparser.expression.StringValue;
 import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
 import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
-import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
 import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
 import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
 import net.sf.jsqlparser.expression.operators.relational.InExpression;
 import net.sf.jsqlparser.expression.operators.relational.MinorThan;
 import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
+import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
 import net.sf.jsqlparser.schema.Column;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -539,12 +539,10 @@ public class ChatQueryServiceImpl implements ChatQueryService {
     @Override
     public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
         SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
-        SemanticSchema semanticSchema = schemaService.getSemanticSchema();
+        DimensionResp dimensionResp = schemaService.getDimension(dimensionValueReq.getElementID());
-        SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
+        Set<Long> dataSetIds = dimensionValueReq.getDataSetIds();
-        Set<Long> detectDataSetIds = new HashSet<>();
+        dimensionValueReq.setModelId(dimensionResp.getModelId());
-        detectDataSetIds.add(schemaElement.getDataSet());
+        List<String> dimensionValues = getDimensionValues(dimensionValueReq, dataSetIds);
-        dimensionValueReq.setModelId(schemaElement.getModel());
-        List<String> dimensionValues = getDimensionValues(dimensionValueReq, detectDataSetIds);
         // if the search results is null,search dimensionValue from database
         if (CollectionUtils.isEmpty(dimensionValues)) {
             semanticQueryResp = queryDatabase(dimensionValueReq, user);
@@ -593,21 +591,12 @@ public class ChatQueryServiceImpl implements ChatQueryService {
                 .collect(Collectors.toList());
     }
 
-    private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) throws Exception {
+    private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) {
-        QueryStructReq queryStructReq = new QueryStructReq();
+        QueryDimValueReq queryDimValueReq = new QueryDimValueReq();
-
+        queryDimValueReq.setValue(dimensionValueReq.getValue());
-        DateConf dateConf = new DateConf();
+        queryDimValueReq.setModelId(dimensionValueReq.getModelId());
-        dateConf.setDateMode(DateConf.DateMode.RECENT);
+        queryDimValueReq.setDimensionBizName(dimensionValueReq.getBizName());
-        dateConf.setUnit(1);
+        return semanticLayerService.queryDimValue(queryDimValueReq, user);
-        dateConf.setPeriod("DAY");
-        queryStructReq.setDateInfo(dateConf);
-        queryStructReq.setLimit(20L);
-        queryStructReq.setDataSetId(dimensionValueReq.getModelId());
-        queryStructReq.setQueryType(QueryType.ID);
-        List<String> groups = new ArrayList<>();
-        groups.add(dimensionValueReq.getBizName());
-        queryStructReq.setGroups(groups);
-        return semanticLayerService.queryByReq(queryStructReq, user);
     }
 
     public void correct(QuerySqlReq querySqlReq, User user) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -48,6 +48,13 @@ import com.tencent.supersonic.headless.server.utils.QueryUtils;
 import com.tencent.supersonic.headless.server.utils.StatUtils;
 import com.tencent.supersonic.headless.server.web.service.DataSetService;
 import com.tencent.supersonic.headless.server.web.service.SchemaService;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.BeanUtils;
+import org.springframework.stereotype.Service;
+
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
@@ -56,12 +63,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.BeanUtils;
-import org.springframework.stereotype.Service;
 
 
 @Service
@@ -246,13 +247,17 @@ public class S2SemanticLayerService implements SemanticLayerService {
         DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getDimensionBizName(),
                 queryDimValueReq.getModelId());
         ModelResp modelResp = modelResps.get(0);
-        String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName());
+        String sql = String.format("select distinct %s from %s where 1=1",
+                dimensionResp.getName(), modelResp.getName());
         List<Dim> timeDims = modelResp.getTimeDimension();
         if (CollectionUtils.isNotEmpty(timeDims)) {
-            sql = String.format("%s where %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
+            sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
                     queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
                     queryDimValueReq.getDateInfo().getEndDate());
         }
+        if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
+            sql += " AND " + queryDimValueReq.getDimensionBizName() + " LIKE '%" + queryDimValueReq.getValue() + "%'";
+        }
         querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
         querySqlReq.setSql(sql);
         return querySqlReq;

headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java

@@ -40,6 +40,8 @@ public interface SchemaService {
 
     DimensionResp getDimension(String bizName, Long modelId);
 
+    DimensionResp getDimension(Long id);
+
     List<MetricResp> getMetrics(MetaFilter metaFilter);
 
     List<DomainResp> getDomainList(User user);

headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java

@@ -317,6 +317,11 @@ public class SchemaServiceImpl implements SchemaService {
         return dimensionService.getDimension(bizName, modelId);
     }
 
+    @Override
+    public DimensionResp getDimension(Long id) {
+        return dimensionService.getDimension(id);
+    }
+
     @Override
     public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
         return dimensionService.getDimensions(metaFilter);