diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java index ea6b88363..9bd6563aa 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java @@ -49,7 +49,8 @@ public class EmbeddingServiceImpl implements EmbeddingService { try { EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel(); Embedding embedding = embeddingModel.embed(question).content(); - boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding); + boolean existSegment = + existSegment(collectionName, embeddingStore, query, embedding); if (existSegment) { continue; } @@ -62,14 +63,14 @@ public class EmbeddingServiceImpl implements EmbeddingService { } } - private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query, - Embedding embedding) { + private boolean existSegment(String collectionName, EmbeddingStore embeddingStore, + TextSegment query, Embedding embedding) { String queryId = TextSegmentConvert.getQueryId(query); if (queryId == null) { return false; } // Check cache first - Boolean cachedResult = cache.getIfPresent(collectionName+queryId); + Boolean cachedResult = cache.getIfPresent(collectionName + queryId); if (cachedResult != null) { return cachedResult; } @@ -82,7 +83,7 @@ public class EmbeddingServiceImpl implements EmbeddingService { EmbeddingSearchResult result = embeddingStore.search(request); List> relevant = result.matches(); boolean exists = CollectionUtils.isNotEmpty(relevant); - cache.put(collectionName+queryId, exists); + cache.put(collectionName + queryId, exists); return exists; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java index f385c73fc..e9efbcf3e 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java @@ -30,6 +30,7 @@ public class QueryDataSetReq { private List metricFilters = new ArrayList<>(); private DateConf dateInfo; private Long limit = 2000L; + private Long offset = 0L; private QueryType queryType = QueryType.DETAIL; private boolean innerLayerNative = false; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 5a6c3816e..03b85f992 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -3,11 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; @@ -25,12 +21,7 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; -import net.sf.jsqlparser.statement.select.GroupByElement; -import net.sf.jsqlparser.statement.select.Limit; -import net.sf.jsqlparser.statement.select.OrderByElement; -import net.sf.jsqlparser.statement.select.ParenthesedSelect; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.SelectItem; +import net.sf.jsqlparser.statement.select.*; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -52,6 +43,7 @@ public class QueryStructReq extends SemanticQueryReq { private List metricFilters = new ArrayList<>(); private DateConf dateInfo; private long limit = Constants.DEFAULT_DETAIL_LIMIT; + private long offset; private QueryType queryType = QueryType.DETAIL; private boolean convertToSql = true; @@ -170,12 +162,15 @@ public class QueryStructReq extends SemanticQueryReq { // 5. Set the limit clause plainSelect.setLimit(buildLimit(queryStructReq)); - // 6. Set having clause + // 6. Set the offset clause + plainSelect.setOffset(buildOffset(queryStructReq)); + + // 7. Set the having clause plainSelect.setHaving(buildHavingClause(queryStructReq)); select.setSelect(plainSelect); - // 6. Set where clause + // 8. Set the where clause return addWhereClauses(select.toString(), queryStructReq, isBizName); } @@ -262,6 +257,15 @@ public class QueryStructReq extends SemanticQueryReq { return limit; } + private Offset buildOffset(QueryStructReq queryStructReq) { + if (Objects.isNull(queryStructReq.getOffset())) { + return null; + } + Offset offset = new Offset(); + offset.setOffset(new LongValue(queryStructReq.getOffset())); + return offset; + } + private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName) throws JSQLParserException { SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java index da2f22c47..d7c9f058d 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java @@ -25,7 +25,7 @@ public class RuleSqlCorrector extends BaseSemanticCorrector { @Override public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class); - if(!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) { + if (!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) { return; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java index 04b4d31d4..6c02674f5 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java @@ -56,7 +56,6 @@ public abstract class BaseMatchStrategy implements MatchStr for (T oneRoundResult : oneRoundResults) { if (existResults.contains(oneRoundResult)) { boolean isDeleted = existResults.removeIf(existResult -> { -// boolean delete = existResult.lessSimilar(oneRoundResult); boolean delete = existResult.lessOrEqualSimilar(oneRoundResult); if (delete) { log.debug("deleted existResult:{}", existResult); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java index accbf63ee..ab8c7a0eb 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java @@ -59,7 +59,8 @@ public class ParserConfig extends ParameterConfig { @Override public List getSysParameters() { - return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER, - PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD); + return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, + PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, + PARSER_FIELDS_COUNT_THRESHOLD); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java index f6e9489ef..74b9ce019 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java @@ -20,6 +20,7 @@ public class StructQuery { private List metricFilters = new ArrayList(); private DateConf dateInfo; private Long limit = 2000L; + private Long offset = 0L; private QueryType queryType; private List params = new ArrayList<>(); }