(improvement)(chat) dsl supports revision and fix queryDimensionValue performance (#188)

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java

@@ -4,6 +4,10 @@ import lombok.Data;
 
 @Data
 public class DimensionValueReq {
+    private Integer agentId;
+
+    private Long elementID;
+
     private Long modelId;
 
     private String bizName;

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -16,9 +16,9 @@ import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
+import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
-import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
@@ -32,31 +32,39 @@ import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
 import com.tencent.supersonic.chat.responder.parse.ParseResponder;
 import com.tencent.supersonic.chat.service.ChatService;
 import com.tencent.supersonic.chat.service.QueryService;
+import com.tencent.supersonic.common.pojo.QueryColumn;
+import com.tencent.supersonic.knowledge.dictionary.MapResult;
+import com.tencent.supersonic.knowledge.service.SearchService;
 import com.tencent.supersonic.chat.service.StatisticsService;
 import com.tencent.supersonic.chat.utils.ComponentFactory;
-import java.util.Map;
-import com.tencent.supersonic.chat.utils.SolvedQueryManager;
-import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
-import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
+
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Objects;
 import java.util.Set;
 import java.util.HashSet;
-import java.util.HashMap;
 import java.util.Comparator;
-import java.util.Objects;
+
+import com.tencent.supersonic.chat.utils.SolvedQueryManager;
+
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
+import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
+
 import java.util.stream.Collectors;
+
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Primary;
@@ -75,6 +83,8 @@ public class QueryServiceImpl implements QueryService {
     private StatisticsService statisticsService;
     @Autowired
     private SolvedQueryManager solvedQueryManager;
+    @Autowired
+    private SearchService searchService;
 
     @Value("${time.threshold: 100}")
     private Integer timeThreshold;
@@ -195,8 +205,8 @@ public class QueryServiceImpl implements QueryService {
     }
 
     public void saveInfo(List<StatisticsDO> timeCostDOList,
-            String queryText, Long queryId,
-            String userName, Long chatId) {
+                         String queryText, Long queryId,
+                         String userName, Long chatId) {
         List<StatisticsDO> list = timeCostDOList.stream()
                 .filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList());
         list.forEach(o -> {
@@ -264,6 +274,7 @@ public class QueryServiceImpl implements QueryService {
         ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(),
                 queryData.getUser().getName(), queryData.getParseId());
         SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
+        SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
         if (!parseInfo.getQueryMode().equals(DslQuery.QUERY_MODE)) {
             if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {
                 parseInfo.setDimensions(queryData.getDimensions());
@@ -291,24 +302,36 @@ public class QueryServiceImpl implements QueryService {
             for (QueryFilter dslQueryFilter : queryData.getDimensionFilters()) {
                 Map<String, String> map = new HashMap<>();
                 for (FilterExpression filterExpression : filterExpressionList) {
-                    if (filterExpression.getFieldName().equals(dslQueryFilter.getBizName())
+                    if (filterExpression.getFieldName() != null
+                            && filterExpression.getFieldName().equals(dslQueryFilter.getName())
                             && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) {
                         map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString());
+                        parseInfo.getDimensionFilters().stream().forEach(o -> {
+                            if (o.getName().equals(dslQueryFilter.getName())) {
+                                o.setValue(dslQueryFilter.getValue());
+                            }
+                        });
                         break;
                     }
                 }
-                filedNameToValueMap.put(dslQueryFilter.getBizName(), map);
+                filedNameToValueMap.put(dslQueryFilter.getName(), map);
             }
             for (QueryFilter dslQueryFilter : queryData.getMetricFilters()) {
                 Map<String, String> map = new HashMap<>();
                 for (FilterExpression filterExpression : filterExpressionList) {
-                    if (filterExpression.getFieldName().equals(dslQueryFilter.getBizName())
+                    if (filterExpression.getFieldName() != null
+                            && filterExpression.getFieldName().equals(dslQueryFilter.getName())
                             && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) {
                         map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString());
+                        parseInfo.getMetricFilters().stream().forEach(o -> {
+                            if (o.getName().equals(dslQueryFilter.getName())) {
+                                o.setValue(dslQueryFilter.getValue());
+                            }
+                        });
                         break;
                     }
                 }
-                filedNameToValueMap.put(dslQueryFilter.getBizName(), map);
+                filedNameToValueMap.put(dslQueryFilter.getName(), map);
             }
             String dateField = "sys_imp_date";
             if (Objects.nonNull(queryData.getDateInfo())) {
@@ -316,7 +339,8 @@ public class QueryServiceImpl implements QueryService {
                 List<String> dateFields = Lists.newArrayList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month");
                 if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) {
                     for (FilterExpression filterExpression : filterExpressionList) {
-                        if (dateFields.contains(filterExpression.getFieldName())) {
+                        if (filterExpression.getFieldName() != null
+                                && dateFields.contains(filterExpression.getFieldName())) {
                             dateField = filterExpression.getFieldName();
                             map.put(filterExpression.getFieldValue().toString(),
                                     queryData.getDateInfo().getStartDate());
@@ -341,6 +365,7 @@ public class QueryServiceImpl implements QueryService {
                     }
                 }
                 filedNameToValueMap.put(dateField, map);
+                parseInfo.setDateInfo(queryData.getDateInfo());
             }
             log.info("filedNameToValueMap:{}", filedNameToValueMap);
             correctorSql = SqlParserUpdateHelper.replaceValue(correctorSql, filedNameToValueMap);
@@ -350,8 +375,13 @@ public class QueryServiceImpl implements QueryService {
             Map<String, Object> properties = new HashMap<>();
             properties.put(Constants.CONTEXT, dslParseResult);
             parseInfo.setProperties(properties);
+            parseInfo.getSqlInfo().setLogicSql(correctorSql);
+            semanticQuery.setParseInfo(parseInfo);
+            ExplainResp explain = semanticQuery.explain(user);
+            if (!Objects.isNull(explain)) {
+                parseInfo.getSqlInfo().setQuerySql(explain.getSql());
+            }
         }
-        SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
         semanticQuery.setParseInfo(parseInfo);
         QueryResult queryResult = semanticQuery.execute(user);
         queryResult.setChatContext(semanticQuery.getParseInfo());
@@ -373,27 +403,47 @@ public class QueryServiceImpl implements QueryService {
         List<String> groups = new ArrayList<>();
         groups.add(dimensionValueReq.getBizName());
         queryStructReq.setGroups(groups);
-        if (!Objects.isNull(dimensionValueReq.getValue())) {
-            List<Filter> dimensionFilters = new ArrayList<>();
-            Filter dimensionFilter = new Filter();
-            dimensionFilter.setOperator(FilterOperatorEnum.LIKE);
-            dimensionFilter.setRelation(Filter.Relation.FILTER);
-            dimensionFilter.setBizName(dimensionValueReq.getBizName());
-            dimensionFilter.setValue(dimensionValueReq.getValue());
-            dimensionFilters.add(dimensionFilter);
-            queryStructReq.setDimensionFilters(dimensionFilters);
+        if ((!Objects.isNull(dimensionValueReq.getValue()))
+                && StringUtils.isNotBlank(dimensionValueReq.getValue().toString())) {
+            return queryHanlpDimensionValue(dimensionValueReq, user);
         }
         SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
         QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
-        Set<String> dimensionValues = new HashSet<>();
-        queryResultWithSchemaResp.getResultList().removeIf(o -> {
-            if (dimensionValues.contains(o.get(dimensionValueReq.getBizName()))) {
-                return true;
-            } else {
-                dimensionValues.add(o.get(dimensionValueReq.getBizName()).toString());
-                return false;
+        return queryResultWithSchemaResp;
+    }
+
+    public Object queryHanlpDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
+        QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp();
+        Set<Long> detectModelIds = new HashSet<>();
+        detectModelIds.add(dimensionValueReq.getModelId());
+        List<MapResult> mapResultList = SearchService.prefixSearch(dimensionValueReq.getValue().toString(),
+                2000, dimensionValueReq.getAgentId(), detectModelIds);
+        log.info("mapResultList:{}", mapResultList);
+        mapResultList = mapResultList.stream().filter(o -> {
+            for (String nature : o.getNatures()) {
+                String[] natureArray = nature.split("_");
+                if (natureArray[2].equals(dimensionValueReq.getElementID().toString())) {
+                    return true;
+                }
             }
+            return false;
+        }).collect(Collectors.toList());
+        log.info("mapResultList:{}", mapResultList);
+        List<QueryColumn> columns = new ArrayList<>();
+        QueryColumn queryColumn = new QueryColumn();
+        queryColumn.setNameEn(dimensionValueReq.getBizName());
+        queryColumn.setShowType("CATEGORY");
+        queryColumn.setAuthorized(true);
+        queryColumn.setType("CHAR");
+        columns.add(queryColumn);
+        List<Map<String, Object>> resultList = new ArrayList<>();
+        mapResultList.stream().forEach(o -> {
+            Map<String, Object> map = new HashMap<>();
+            map.put(dimensionValueReq.getBizName(), o.getName());
+            resultList.add(map);
         });
+        queryResultWithSchemaResp.setColumns(columns);
+        queryResultWithSchemaResp.setResultList(resultList);
         return queryResultWithSchemaResp;
     }
 

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticInterpreter.java

@@ -17,13 +17,14 @@ import org.springframework.util.CollectionUtils;
 public abstract class BaseSemanticInterpreter implements SemanticInterpreter {
 
     protected final Cache<String, List<ModelSchemaResp>> modelSchemaCache =
-            CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build();
+            CacheBuilder.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS).build();
 
     @SneakyThrows
     public List<ModelSchemaResp> fetchModelSchema(List<Long> ids, Boolean cacheEnable) {
         if (cacheEnable) {
             return modelSchemaCache.get(String.valueOf(ids), () -> {
                 List<ModelSchemaResp> data = doFetchModelSchema(ids);
+                modelSchemaCache.put(String.valueOf(ids), data);
                 return data;
             });
         }