Merge de427b8093 into 2e81b190a4

common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java

@@ -49,8 +49,7 @@ public class EmbeddingServiceImpl implements EmbeddingService {
             try {
                 EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
                 Embedding embedding = embeddingModel.embed(question).content();
-                boolean existSegment =
+                boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding);
-                        existSegment(collectionName, embeddingStore, query, embedding);
                 if (existSegment) {
                     continue;
                 }
@@ -63,8 +62,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
         }
     }
 
-    private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
+    private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query,
-            TextSegment query, Embedding embedding) {
+            Embedding embedding) {
         String queryId = TextSegmentConvert.getQueryId(query);
         if (queryId == null) {
             return false;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java

@@ -30,7 +30,6 @@ public class QueryDataSetReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private Long limit = 2000L;
-    private Long offset = 0L;
     private QueryType queryType = QueryType.DETAIL;
     private boolean innerLayerNative = false;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -3,7 +3,11 @@ package com.tencent.supersonic.headless.api.pojo.request;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.pojo.*;
+import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.Filter;
+import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -21,7 +25,12 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.schema.Table;
-import net.sf.jsqlparser.statement.select.*;
+import net.sf.jsqlparser.statement.select.GroupByElement;
+import net.sf.jsqlparser.statement.select.Limit;
+import net.sf.jsqlparser.statement.select.OrderByElement;
+import net.sf.jsqlparser.statement.select.ParenthesedSelect;
+import net.sf.jsqlparser.statement.select.PlainSelect;
+import net.sf.jsqlparser.statement.select.SelectItem;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -43,7 +52,6 @@ public class QueryStructReq extends SemanticQueryReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private long limit = Constants.DEFAULT_DETAIL_LIMIT;
-    private long offset;
     private QueryType queryType = QueryType.DETAIL;
     private boolean convertToSql = true;
 
@@ -162,15 +170,12 @@ public class QueryStructReq extends SemanticQueryReq {
         // 5. Set the limit clause
         plainSelect.setLimit(buildLimit(queryStructReq));
 
-        // 6. Set the offset clause
+        // 6. Set having clause
-        plainSelect.setOffset(buildOffset(queryStructReq));
-
-        // 7. Set the having clause
         plainSelect.setHaving(buildHavingClause(queryStructReq));
 
         select.setSelect(plainSelect);
 
-        // 8. Set the where clause
+        // 6. Set where clause
         return addWhereClauses(select.toString(), queryStructReq, isBizName);
     }
 
@@ -257,15 +262,6 @@ public class QueryStructReq extends SemanticQueryReq {
         return limit;
     }
 
-    private Offset buildOffset(QueryStructReq queryStructReq) {
-        if (Objects.isNull(queryStructReq.getOffset())) {
-            return null;
-        }
-        Offset offset = new Offset();
-        offset.setOffset(new LongValue(queryStructReq.getOffset()));
-        return offset;
-    }
-
     private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName)
             throws JSQLParserException {
         SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java

@@ -22,9 +22,4 @@ public abstract class MapResult implements Serializable {
         return this.getMapKey().equals(otherResult.getMapKey())
                 && this.similarity < otherResult.similarity;
     }
-
-    public Boolean lessOrEqualSimilar(MapResult otherResult) {
-        return this.getMapKey().equals(otherResult.getMapKey())
-                && this.similarity <= otherResult.similarity;
-    }
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java

@@ -75,8 +75,6 @@ public class MetaEmbeddingService {
             return dataSetIds.stream().map(dataSetId -> {
                 Retrieval newRetrieval = new Retrieval();
                 BeanUtils.copyProperties(retrieval, newRetrieval);
-                HashMap<String, Object> newMetadata = new HashMap<>(retrieval.getMetadata());
-                newRetrieval.setMetadata(newMetadata);
                 newRetrieval.getMetadata().putIfAbsent("dataSetId",
                         dataSetId + Constants.UNDERLINE);
                 return newRetrieval;

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java

@@ -56,7 +56,7 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
         for (T oneRoundResult : oneRoundResults) {
             if (existResults.contains(oneRoundResult)) {
                 boolean isDeleted = existResults.removeIf(existResult -> {
-                    boolean delete = existResult.lessOrEqualSimilar(oneRoundResult);
+                    boolean delete = existResult.lessSimilar(oneRoundResult);
                     if (delete) {
                         log.debug("deleted existResult:{}", existResult);
                     }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java

@@ -59,8 +59,7 @@ public class ParserConfig extends ParameterConfig {
 
     @Override
     public List<Parameter> getSysParameters() {
-        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE,
+        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER,
-                PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT,
+                PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD);
-                PARSER_FIELDS_COUNT_THRESHOLD);
     }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java

@@ -20,7 +20,6 @@ public class StructQuery {
     private List<Filter> metricFilters = new ArrayList();
     private DateConf dateInfo;
     private Long limit = 2000L;
-    private Long offset = 0L;
     private QueryType queryType;
     private List<Param> params = new ArrayList<>();
 }