topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature)(headless)Support offset clause in struct query.
Some checks failed
supersonic CentOS CI / build (21) (push) Has been cancelled
Details
supersonic mac CI / build (21) (push) Has been cancelled
Details
supersonic ubuntu CI / build (21) (push) Has been cancelled
Details
supersonic windows CI / build (21) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
jerryjzhang
2025-03-23 14:30:56 +08:00
parent db8f340e2d
commit d2a43a99c8
7 changed files with 29 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java
View File

@@ -49,7 +49,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
try { try {
EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(); EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
Embedding embedding = embeddingModel.embed(question).content(); Embedding embedding = embeddingModel.embed(question).content();
boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding); boolean existSegment =
existSegment(collectionName, embeddingStore, query, embedding);
if (existSegment) { if (existSegment) {
continue; continue;
} }
@@ -62,14 +63,14 @@ public class EmbeddingServiceImpl implements EmbeddingService {
} }
} }
private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query, private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
Embedding embedding) { TextSegment query, Embedding embedding) {
String queryId = TextSegmentConvert.getQueryId(query); String queryId = TextSegmentConvert.getQueryId(query);
if (queryId == null) { if (queryId == null) {
return false; return false;
} }
// Check cache first // Check cache first
Boolean cachedResult = cache.getIfPresent(collectionName+queryId); Boolean cachedResult = cache.getIfPresent(collectionName + queryId);
if (cachedResult != null) { if (cachedResult != null) {
return cachedResult; return cachedResult;
} }
@@ -82,7 +83,7 @@ public class EmbeddingServiceImpl implements EmbeddingService {
EmbeddingSearchResult result = embeddingStore.search(request); EmbeddingSearchResult result = embeddingStore.search(request);
List<EmbeddingMatch<TextSegment>> relevant = result.matches(); List<EmbeddingMatch<TextSegment>> relevant = result.matches();
boolean exists = CollectionUtils.isNotEmpty(relevant); boolean exists = CollectionUtils.isNotEmpty(relevant);
cache.put(collectionName+queryId, exists); cache.put(collectionName + queryId, exists);
return exists; return exists;
} }

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java
View File

@@ -30,6 +30,7 @@ public class QueryDataSetReq {
private List<Filter> metricFilters = new ArrayList<>(); private List<Filter> metricFilters = new ArrayList<>();
private DateConf dateInfo; private DateConf dateInfo;
private Long limit = 2000L; private Long limit = 2000L;
private Long offset = 0L;
private QueryType queryType = QueryType.DETAIL; private QueryType queryType = QueryType.DETAIL;
private boolean innerLayerNative = false; private boolean innerLayerNative = false;
} }

30
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -3,11 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.*;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
@@ -25,12 +21,7 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.select.GroupByElement; import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.statement.select.Limit;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.SelectItem;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -52,6 +43,7 @@ public class QueryStructReq extends SemanticQueryReq {
private List<Filter> metricFilters = new ArrayList<>(); private List<Filter> metricFilters = new ArrayList<>();
private DateConf dateInfo; private DateConf dateInfo;
private long limit = Constants.DEFAULT_DETAIL_LIMIT; private long limit = Constants.DEFAULT_DETAIL_LIMIT;
private long offset;
private QueryType queryType = QueryType.DETAIL; private QueryType queryType = QueryType.DETAIL;
private boolean convertToSql = true; private boolean convertToSql = true;
@@ -170,12 +162,15 @@ public class QueryStructReq extends SemanticQueryReq {
// 5. Set the limit clause // 5. Set the limit clause
plainSelect.setLimit(buildLimit(queryStructReq)); plainSelect.setLimit(buildLimit(queryStructReq));
// 6. Set having clause // 6. Set the offset clause
plainSelect.setOffset(buildOffset(queryStructReq));
// 7. Set the having clause
plainSelect.setHaving(buildHavingClause(queryStructReq)); plainSelect.setHaving(buildHavingClause(queryStructReq));
select.setSelect(plainSelect); select.setSelect(plainSelect);
// 6. Set where clause // 8. Set the where clause
return addWhereClauses(select.toString(), queryStructReq, isBizName); return addWhereClauses(select.toString(), queryStructReq, isBizName);
} }
@@ -262,6 +257,15 @@ public class QueryStructReq extends SemanticQueryReq {
return limit; return limit;
} }
private Offset buildOffset(QueryStructReq queryStructReq) {
if (Objects.isNull(queryStructReq.getOffset())) {
return null;
}
Offset offset = new Offset();
offset.setOffset(new LongValue(queryStructReq.getOffset()));
return offset;
}
private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName) private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName)
throws JSQLParserException { throws JSQLParserException {
SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class); SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java
View File

@@ -25,7 +25,7 @@ public class RuleSqlCorrector extends BaseSemanticCorrector {
@Override @Override
public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class); ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
if(!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) { if (!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) {
return; return;
} }

1
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
View File

@@ -56,7 +56,6 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
for (T oneRoundResult : oneRoundResults) { for (T oneRoundResult : oneRoundResults) {
if (existResults.contains(oneRoundResult)) { if (existResults.contains(oneRoundResult)) {
boolean isDeleted = existResults.removeIf(existResult -> { boolean isDeleted = existResults.removeIf(existResult -> {
// boolean delete = existResult.lessSimilar(oneRoundResult);
boolean delete = existResult.lessOrEqualSimilar(oneRoundResult); boolean delete = existResult.lessOrEqualSimilar(oneRoundResult);
if (delete) { if (delete) {
log.debug("deleted existResult:{}", existResult); log.debug("deleted existResult:{}", existResult);

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java
View File

@@ -59,7 +59,8 @@ public class ParserConfig extends ParameterConfig {
@Override @Override
public List<Parameter> getSysParameters() { public List<Parameter> getSysParameters() {
return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER, return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE,
PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD); PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT,
PARSER_FIELDS_COUNT_THRESHOLD);
} }
} }

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java
View File

@@ -20,6 +20,7 @@ public class StructQuery {
private List<Filter> metricFilters = new ArrayList(); private List<Filter> metricFilters = new ArrayList();
private DateConf dateInfo; private DateConf dateInfo;
private Long limit = 2000L; private Long limit = 2000L;
private Long offset = 0L;
private QueryType queryType; private QueryType queryType;
private List<Param> params = new ArrayList<>(); private List<Param> params = new ArrayList<>();
} }