(feature)(headless)Support offset clause in struct query.

2025-12-10 11:07:06 +00:00 · 2025-03-23 14:30:56 +08:00
parent db8f340e2d
commit d2a43a99c8
7 changed files with 29 additions and 22 deletions
--- a/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java
+++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java
@@ -49,7 +49,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
            try {
                EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
                Embedding embedding = embeddingModel.embed(question).content();
-                boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding);
+                boolean existSegment =
+                        existSegment(collectionName, embeddingStore, query, embedding);
                if (existSegment) {
                    continue;
                }
@@ -62,14 +63,14 @@ public class EmbeddingServiceImpl implements EmbeddingService {
        }
    }

-    private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query,
-            Embedding embedding) {
+    private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
+            TextSegment query, Embedding embedding) {
        String queryId = TextSegmentConvert.getQueryId(query);
        if (queryId == null) {
            return false;
        }
        // Check cache first
-        Boolean cachedResult = cache.getIfPresent(collectionName+queryId);
+        Boolean cachedResult = cache.getIfPresent(collectionName + queryId);
        if (cachedResult != null) {
            return cachedResult;
        }
@@ -82,7 +83,7 @@ public class EmbeddingServiceImpl implements EmbeddingService {
        EmbeddingSearchResult result = embeddingStore.search(request);
        List<EmbeddingMatch<TextSegment>> relevant = result.matches();
        boolean exists = CollectionUtils.isNotEmpty(relevant);
-        cache.put(collectionName+queryId, exists);
+        cache.put(collectionName + queryId, exists);
        return exists;
    }

--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java
@@ -30,6 +30,7 @@ public class QueryDataSetReq {
    private List<Filter> metricFilters = new ArrayList<>();
    private DateConf dateInfo;
    private Long limit = 2000L;
+    private Long offset = 0L;
    private QueryType queryType = QueryType.DETAIL;
    private boolean innerLayerNative = false;
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
@@ -3,11 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.Order;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -25,12 +21,7 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.schema.Table;
-import net.sf.jsqlparser.statement.select.GroupByElement;
-import net.sf.jsqlparser.statement.select.Limit;
-import net.sf.jsqlparser.statement.select.OrderByElement;
-import net.sf.jsqlparser.statement.select.ParenthesedSelect;
-import net.sf.jsqlparser.statement.select.PlainSelect;
-import net.sf.jsqlparser.statement.select.SelectItem;
+import net.sf.jsqlparser.statement.select.*;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -52,6 +43,7 @@ public class QueryStructReq extends SemanticQueryReq {
    private List<Filter> metricFilters = new ArrayList<>();
    private DateConf dateInfo;
    private long limit = Constants.DEFAULT_DETAIL_LIMIT;
+    private long offset;
    private QueryType queryType = QueryType.DETAIL;
    private boolean convertToSql = true;

@@ -170,12 +162,15 @@ public class QueryStructReq extends SemanticQueryReq {
        // 5. Set the limit clause
        plainSelect.setLimit(buildLimit(queryStructReq));

-        // 6. Set having clause
+        // 6. Set the offset clause
+        plainSelect.setOffset(buildOffset(queryStructReq));
+
+        // 7. Set the having clause
        plainSelect.setHaving(buildHavingClause(queryStructReq));

        select.setSelect(plainSelect);

-        // 6. Set where clause
+        // 8. Set the where clause
        return addWhereClauses(select.toString(), queryStructReq, isBizName);
    }

@@ -262,6 +257,15 @@ public class QueryStructReq extends SemanticQueryReq {
        return limit;
    }

+    private Offset buildOffset(QueryStructReq queryStructReq) {
+        if (Objects.isNull(queryStructReq.getOffset())) {
+            return null;
+        }
+        Offset offset = new Offset();
+        offset.setOffset(new LongValue(queryStructReq.getOffset()));
+        return offset;
+    }
+
    private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName)
            throws JSQLParserException {
        SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java
@@ -25,7 +25,7 @@ public class RuleSqlCorrector extends BaseSemanticCorrector {
    @Override
    public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
        ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
-        if(!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) {
+        if (!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) {
            return;
        }

--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
@@ -56,7 +56,6 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
        for (T oneRoundResult : oneRoundResults) {
            if (existResults.contains(oneRoundResult)) {
                boolean isDeleted = existResults.removeIf(existResult -> {
-//                    boolean delete = existResult.lessSimilar(oneRoundResult);
                    boolean delete = existResult.lessOrEqualSimilar(oneRoundResult);
                    if (delete) {
                        log.debug("deleted existResult:{}", existResult);
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java
@@ -59,7 +59,8 @@ public class ParserConfig extends ParameterConfig {

    @Override
    public List<Parameter> getSysParameters() {
-        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER,
-                PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD);
+        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE,
+                PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT,
+                PARSER_FIELDS_COUNT_THRESHOLD);
    }
 }
--- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java
+++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java
@@ -20,6 +20,7 @@ public class StructQuery {
    private List<Filter> metricFilters = new ArrayList();
    private DateConf dateInfo;
    private Long limit = 2000L;
+    private Long offset = 0L;
    private QueryType queryType;
    private List<Param> params = new ArrayList<>();
 }