topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(fix)(Chat) Fix dimension value query failed (#1257)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-06-27 22:27:09 +08:00
committed by GitHub
parent 391c0dccc8
commit 995c4f8ada
7 changed files with 54 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java
View File

@@ -17,7 +17,6 @@ import com.tencent.supersonic.chat.server.service.ChatService;
import com.tencent.supersonic.chat.server.util.ComponentFactory;
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
@@ -45,6 +44,8 @@ public class ChatServiceImpl implements ChatService {
private ChatQueryService chatQueryService;
@Autowired
private RetrieveService retrieveService;
@Autowired
private AgentService agentService;
private List<ChatParser> chatParsers = ComponentFactory.getChatParsers();
private List<ChatExecutor> chatExecutors = ComponentFactory.getChatExecutors();
private List<ParseResultProcessor> parseResultProcessors = ComponentFactory.getParseProcessors();
@@ -111,7 +112,6 @@ public class ChatServiceImpl implements ChatService {
private ChatParseContext buildParseContext(ChatParseReq chatParseReq) {
ChatParseContext chatParseContext = new ChatParseContext();
BeanMapper.mapper(chatParseReq, chatParseContext);
AgentService agentService = ContextUtils.getBean(AgentService.class);
Agent agent = agentService.getAgent(chatParseReq.getAgentId());
chatParseContext.setAgent(agent);
QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
@@ -147,6 +147,9 @@ public class ChatServiceImpl implements ChatService {
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
Integer agentId = dimensionValueReq.getAgentId();
Agent agent = agentService.getAgent(agentId);
dimensionValueReq.setDataSetIds(agent.getDataSetIds());
return chatQueryService.queryDimensionValue(dimensionValueReq, user);
}

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java
View File

@@ -1,7 +1,8 @@
package com.tencent.supersonic.headless.api.pojo.request;
import javax.validation.constraints.NotNull;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.Set;
@Data
public class DimensionValueReq {
@@ -17,4 +18,7 @@ public class DimensionValueReq {
@NotNull
private String value;
private Set<Long> dataSetIds;
}

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java
View File

@@ -189,7 +189,7 @@ public class SearchService {
String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT
+ dimensionValueReq.getElementID();
PriorityQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
if (org.apache.commons.collections.CollectionUtils.isEmpty(terms)) {
if (CollectionUtils.isEmpty(terms)) {
return new ArrayList<>();
}
return terms.stream().map(term -> term.getWord()).collect(Collectors.toList());

57
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java
View File

@@ -2,7 +2,11 @@ package com.tencent.supersonic.headless.server.facade.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
@@ -10,11 +14,6 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
@@ -32,15 +31,16 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
@@ -48,6 +48,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.chat.ChatContext;
import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector;
import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector;
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
@@ -59,29 +61,27 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.headless.chat.ChatContext;
import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.web.service.ChatContextService;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import com.tencent.supersonic.headless.server.web.service.WorkflowService;
import com.tencent.supersonic.headless.server.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -539,12 +539,10 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
Set<Long> detectDataSetIds = new HashSet<>();
detectDataSetIds.add(schemaElement.getDataSet());
dimensionValueReq.setModelId(schemaElement.getModel());
List<String> dimensionValues = getDimensionValues(dimensionValueReq, detectDataSetIds);
DimensionResp dimensionResp = schemaService.getDimension(dimensionValueReq.getElementID());
Set<Long> dataSetIds = dimensionValueReq.getDataSetIds();
dimensionValueReq.setModelId(dimensionResp.getModelId());
List<String> dimensionValues = getDimensionValues(dimensionValueReq, dataSetIds);
// if the search results is null,search dimensionValue from database
if (CollectionUtils.isEmpty(dimensionValues)) {
semanticQueryResp = queryDatabase(dimensionValueReq, user);
@@ -593,21 +591,12 @@ public class ChatQueryServiceImpl implements ChatQueryService {
.collect(Collectors.toList());
}
private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) throws Exception {
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateConf.DateMode.RECENT);
dateConf.setUnit(1);
dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
queryStructReq.setDataSetId(dimensionValueReq.getModelId());
queryStructReq.setQueryType(QueryType.ID);
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());
queryStructReq.setGroups(groups);
return semanticLayerService.queryByReq(queryStructReq, user);
private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) {
QueryDimValueReq queryDimValueReq = new QueryDimValueReq();
queryDimValueReq.setValue(dimensionValueReq.getValue());
queryDimValueReq.setModelId(dimensionValueReq.getModelId());
queryDimValueReq.setDimensionBizName(dimensionValueReq.getBizName());
return semanticLayerService.queryDimValue(queryDimValueReq, user);
}
public void correct(QuerySqlReq querySqlReq, User user) {

21
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -48,6 +48,13 @@ import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.LinkedHashSet;
@@ -56,12 +63,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@Service
@@ -246,13 +247,17 @@ public class S2SemanticLayerService implements SemanticLayerService {
DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getDimensionBizName(),
queryDimValueReq.getModelId());
ModelResp modelResp = modelResps.get(0);
String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName());
String sql = String.format("select distinct %s from %s where 1=1",
dimensionResp.getName(), modelResp.getName());
List<Dim> timeDims = modelResp.getTimeDimension();
if (CollectionUtils.isNotEmpty(timeDims)) {
sql = String.format("%s where %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
}
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
sql += " AND " + queryDimValueReq.getDimensionBizName() + " LIKE '%" + queryDimValueReq.getValue() + "%'";
}
querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySqlReq.setSql(sql);
return querySqlReq;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java
View File

@@ -40,6 +40,8 @@ public interface SchemaService {
DimensionResp getDimension(String bizName, Long modelId);
DimensionResp getDimension(Long id);
List<MetricResp> getMetrics(MetaFilter metaFilter);
List<DomainResp> getDomainList(User user);

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java
View File

@@ -317,6 +317,11 @@ public class SchemaServiceImpl implements SchemaService {
return dimensionService.getDimension(bizName, modelId);
}
@Override
public DimensionResp getDimension(Long id) {
return dimensionService.getDimension(id);
}
@Override
public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
return dimensionService.getDimensions(metaFilter);