diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java index 9233661af..d969100b6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java @@ -4,6 +4,10 @@ import lombok.Data; @Data public class DimensionValueReq { + private Integer agentId; + + private Long elementID; + private Long modelId; private String bizName; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 370f7b0d5..2a73b1289 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -16,9 +16,9 @@ import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; @@ -32,31 +32,39 @@ import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.QueryService; +import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.knowledge.dictionary.MapResult; +import com.tencent.supersonic.knowledge.service.SearchService; import com.tencent.supersonic.chat.service.StatisticsService; import com.tencent.supersonic.chat.utils.ComponentFactory; -import java.util.Map; -import com.tencent.supersonic.chat.utils.SolvedQueryManager; -import com.tencent.supersonic.common.util.jsqlparser.FilterExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; + import java.util.List; import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.Objects; import java.util.Set; import java.util.HashSet; -import java.util.HashMap; import java.util.Comparator; -import java.util.Objects; + +import com.tencent.supersonic.chat.utils.SolvedQueryManager; + +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; +import com.tencent.supersonic.common.util.jsqlparser.FilterExpression; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; + import java.util.stream.Collectors; + import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; -import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Primary; @@ -75,6 +83,8 @@ public class QueryServiceImpl implements QueryService { private StatisticsService statisticsService; @Autowired private SolvedQueryManager solvedQueryManager; + @Autowired + private SearchService searchService; @Value("${time.threshold: 100}") private Integer timeThreshold; @@ -195,8 +205,8 @@ public class QueryServiceImpl implements QueryService { } public void saveInfo(List timeCostDOList, - String queryText, Long queryId, - String userName, Long chatId) { + String queryText, Long queryId, + String userName, Long chatId) { List list = timeCostDOList.stream() .filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList()); list.forEach(o -> { @@ -264,6 +274,7 @@ public class QueryServiceImpl implements QueryService { ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(), queryData.getUser().getName(), queryData.getParseId()); SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); if (!parseInfo.getQueryMode().equals(DslQuery.QUERY_MODE)) { if (CollectionUtils.isNotEmpty(queryData.getDimensions())) { parseInfo.setDimensions(queryData.getDimensions()); @@ -291,24 +302,36 @@ public class QueryServiceImpl implements QueryService { for (QueryFilter dslQueryFilter : queryData.getDimensionFilters()) { Map map = new HashMap<>(); for (FilterExpression filterExpression : filterExpressionList) { - if (filterExpression.getFieldName().equals(dslQueryFilter.getBizName()) + if (filterExpression.getFieldName() != null + && filterExpression.getFieldName().equals(dslQueryFilter.getName()) && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); + parseInfo.getDimensionFilters().stream().forEach(o -> { + if (o.getName().equals(dslQueryFilter.getName())) { + o.setValue(dslQueryFilter.getValue()); + } + }); break; } } - filedNameToValueMap.put(dslQueryFilter.getBizName(), map); + filedNameToValueMap.put(dslQueryFilter.getName(), map); } for (QueryFilter dslQueryFilter : queryData.getMetricFilters()) { Map map = new HashMap<>(); for (FilterExpression filterExpression : filterExpressionList) { - if (filterExpression.getFieldName().equals(dslQueryFilter.getBizName()) + if (filterExpression.getFieldName() != null + && filterExpression.getFieldName().equals(dslQueryFilter.getName()) && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); + parseInfo.getMetricFilters().stream().forEach(o -> { + if (o.getName().equals(dslQueryFilter.getName())) { + o.setValue(dslQueryFilter.getValue()); + } + }); break; } } - filedNameToValueMap.put(dslQueryFilter.getBizName(), map); + filedNameToValueMap.put(dslQueryFilter.getName(), map); } String dateField = "sys_imp_date"; if (Objects.nonNull(queryData.getDateInfo())) { @@ -316,7 +339,8 @@ public class QueryServiceImpl implements QueryService { List dateFields = Lists.newArrayList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month"); if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) { for (FilterExpression filterExpression : filterExpressionList) { - if (dateFields.contains(filterExpression.getFieldName())) { + if (filterExpression.getFieldName() != null + && dateFields.contains(filterExpression.getFieldName())) { dateField = filterExpression.getFieldName(); map.put(filterExpression.getFieldValue().toString(), queryData.getDateInfo().getStartDate()); @@ -341,6 +365,7 @@ public class QueryServiceImpl implements QueryService { } } filedNameToValueMap.put(dateField, map); + parseInfo.setDateInfo(queryData.getDateInfo()); } log.info("filedNameToValueMap:{}", filedNameToValueMap); correctorSql = SqlParserUpdateHelper.replaceValue(correctorSql, filedNameToValueMap); @@ -350,8 +375,13 @@ public class QueryServiceImpl implements QueryService { Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, dslParseResult); parseInfo.setProperties(properties); + parseInfo.getSqlInfo().setLogicSql(correctorSql); + semanticQuery.setParseInfo(parseInfo); + ExplainResp explain = semanticQuery.explain(user); + if (!Objects.isNull(explain)) { + parseInfo.getSqlInfo().setQuerySql(explain.getSql()); + } } - SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); semanticQuery.setParseInfo(parseInfo); QueryResult queryResult = semanticQuery.execute(user); queryResult.setChatContext(semanticQuery.getParseInfo()); @@ -373,27 +403,47 @@ public class QueryServiceImpl implements QueryService { List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); queryStructReq.setGroups(groups); - if (!Objects.isNull(dimensionValueReq.getValue())) { - List dimensionFilters = new ArrayList<>(); - Filter dimensionFilter = new Filter(); - dimensionFilter.setOperator(FilterOperatorEnum.LIKE); - dimensionFilter.setRelation(Filter.Relation.FILTER); - dimensionFilter.setBizName(dimensionValueReq.getBizName()); - dimensionFilter.setValue(dimensionValueReq.getValue()); - dimensionFilters.add(dimensionFilter); - queryStructReq.setDimensionFilters(dimensionFilters); + if ((!Objects.isNull(dimensionValueReq.getValue())) + && StringUtils.isNotBlank(dimensionValueReq.getValue().toString())) { + return queryHanlpDimensionValue(dimensionValueReq, user); } SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user); - Set dimensionValues = new HashSet<>(); - queryResultWithSchemaResp.getResultList().removeIf(o -> { - if (dimensionValues.contains(o.get(dimensionValueReq.getBizName()))) { - return true; - } else { - dimensionValues.add(o.get(dimensionValueReq.getBizName()).toString()); - return false; + return queryResultWithSchemaResp; + } + + public Object queryHanlpDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { + QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp(); + Set detectModelIds = new HashSet<>(); + detectModelIds.add(dimensionValueReq.getModelId()); + List mapResultList = SearchService.prefixSearch(dimensionValueReq.getValue().toString(), + 2000, dimensionValueReq.getAgentId(), detectModelIds); + log.info("mapResultList:{}", mapResultList); + mapResultList = mapResultList.stream().filter(o -> { + for (String nature : o.getNatures()) { + String[] natureArray = nature.split("_"); + if (natureArray[2].equals(dimensionValueReq.getElementID().toString())) { + return true; + } } + return false; + }).collect(Collectors.toList()); + log.info("mapResultList:{}", mapResultList); + List columns = new ArrayList<>(); + QueryColumn queryColumn = new QueryColumn(); + queryColumn.setNameEn(dimensionValueReq.getBizName()); + queryColumn.setShowType("CATEGORY"); + queryColumn.setAuthorized(true); + queryColumn.setType("CHAR"); + columns.add(queryColumn); + List> resultList = new ArrayList<>(); + mapResultList.stream().forEach(o -> { + Map map = new HashMap<>(); + map.put(dimensionValueReq.getBizName(), o.getName()); + resultList.add(map); }); + queryResultWithSchemaResp.setColumns(columns); + queryResultWithSchemaResp.setResultList(resultList); return queryResultWithSchemaResp; } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticInterpreter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticInterpreter.java index 08e6f0291..9d62012b5 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticInterpreter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticInterpreter.java @@ -17,13 +17,14 @@ import org.springframework.util.CollectionUtils; public abstract class BaseSemanticInterpreter implements SemanticInterpreter { protected final Cache> modelSchemaCache = - CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); + CacheBuilder.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS).build(); @SneakyThrows public List fetchModelSchema(List ids, Boolean cacheEnable) { if (cacheEnable) { return modelSchemaCache.get(String.valueOf(ids), () -> { List data = doFetchModelSchema(ids); + modelSchemaCache.put(String.valueOf(ids), data); return data; }); }