(feature)(headless)Support offset clause in struct query.

common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java

@@ -49,7 +49,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
             try {
                 EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
                 Embedding embedding = embeddingModel.embed(question).content();
-                boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding);
+                boolean existSegment =
+                        existSegment(collectionName, embeddingStore, query, embedding);
                 if (existSegment) {
                     continue;
                 }
@@ -62,8 +63,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
         }
     }
 
-    private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query,
-            Embedding embedding) {
+    private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
+            TextSegment query, Embedding embedding) {
         String queryId = TextSegmentConvert.getQueryId(query);
         if (queryId == null) {
             return false;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java

@@ -30,6 +30,7 @@ public class QueryDataSetReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private Long limit = 2000L;
+    private Long offset = 0L;
     private QueryType queryType = QueryType.DETAIL;
     private boolean innerLayerNative = false;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -3,11 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.Order;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -25,12 +21,7 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.schema.Table;
-import net.sf.jsqlparser.statement.select.GroupByElement;
-import net.sf.jsqlparser.statement.select.Limit;
-import net.sf.jsqlparser.statement.select.OrderByElement;
-import net.sf.jsqlparser.statement.select.ParenthesedSelect;
-import net.sf.jsqlparser.statement.select.PlainSelect;
-import net.sf.jsqlparser.statement.select.SelectItem;
+import net.sf.jsqlparser.statement.select.*;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -52,6 +43,7 @@ public class QueryStructReq extends SemanticQueryReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private long limit = Constants.DEFAULT_DETAIL_LIMIT;
+    private long offset;
     private QueryType queryType = QueryType.DETAIL;
     private boolean convertToSql = true;
 
@@ -170,12 +162,15 @@ public class QueryStructReq extends SemanticQueryReq {
         // 5. Set the limit clause
         plainSelect.setLimit(buildLimit(queryStructReq));
 
-        // 6. Set having clause
+        // 6. Set the offset clause
+        plainSelect.setOffset(buildOffset(queryStructReq));
+
+        // 7. Set the having clause
         plainSelect.setHaving(buildHavingClause(queryStructReq));
 
         select.setSelect(plainSelect);
 
-        // 6. Set where clause
+        // 8. Set the where clause
         return addWhereClauses(select.toString(), queryStructReq, isBizName);
     }
 
@@ -262,6 +257,15 @@ public class QueryStructReq extends SemanticQueryReq {
         return limit;
     }
 
+    private Offset buildOffset(QueryStructReq queryStructReq) {
+        if (Objects.isNull(queryStructReq.getOffset())) {
+            return null;
+        }
+        Offset offset = new Offset();
+        offset.setOffset(new LongValue(queryStructReq.getOffset()));
+        return offset;
+    }
+
     private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName)
             throws JSQLParserException {
         SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java

@@ -56,7 +56,6 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
         for (T oneRoundResult : oneRoundResults) {
             if (existResults.contains(oneRoundResult)) {
                 boolean isDeleted = existResults.removeIf(existResult -> {
-//                    boolean delete = existResult.lessSimilar(oneRoundResult);
                     boolean delete = existResult.lessOrEqualSimilar(oneRoundResult);
                     if (delete) {
                         log.debug("deleted existResult:{}", existResult);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java

@@ -59,7 +59,8 @@ public class ParserConfig extends ParameterConfig {
 
     @Override
     public List<Parameter> getSysParameters() {
-        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER,
-                PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD);
+        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE,
+                PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT,
+                PARSER_FIELDS_COUNT_THRESHOLD);
     }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java

@@ -20,6 +20,7 @@ public class StructQuery {
     private List<Filter> metricFilters = new ArrayList();
     private DateConf dateInfo;
     private Long limit = 2000L;
+    private Long offset = 0L;
     private QueryType queryType;
     private List<Param> params = new ArrayList<>();
 }