Merge f3ffcb94f1 into d2a43a99c8

Merge branch 'tencentmusic:master' into master
(feature)(headless)Support offset clause in struct query.
2026-04-29 04:14:20 +08:00 · 2025-03-23 07:51:23 +00:00 · 2025-03-23 15:51:20 +08:00 · 2025-03-23 14:30:56 +08:00 · 2025-03-23 14:14:08 +08:00 · 2025-03-19 12:23:39 +08:00
12 changed files with 68 additions and 52 deletions
--- a/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java
+++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java
@@ -49,7 +49,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
            try {
                EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
                Embedding embedding = embeddingModel.embed(question).content();
-                boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding);
+                boolean existSegment =
+                        existSegment(collectionName, embeddingStore, query, embedding);
                if (existSegment) {
                    continue;
                }
@@ -62,14 +63,14 @@ public class EmbeddingServiceImpl implements EmbeddingService {
        }
    }

-    private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query,
-            Embedding embedding) {
+    private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
+            TextSegment query, Embedding embedding) {
        String queryId = TextSegmentConvert.getQueryId(query);
        if (queryId == null) {
            return false;
        }
        // Check cache first
-        Boolean cachedResult = cache.getIfPresent(collectionName+queryId);
+        Boolean cachedResult = cache.getIfPresent(collectionName + queryId);
        if (cachedResult != null) {
            return cachedResult;
        }
@@ -82,7 +83,7 @@ public class EmbeddingServiceImpl implements EmbeddingService {
        EmbeddingSearchResult result = embeddingStore.search(request);
        List<EmbeddingMatch<TextSegment>> relevant = result.matches();
        boolean exists = CollectionUtils.isNotEmpty(relevant);
-        cache.put(collectionName+queryId, exists);
+        cache.put(collectionName + queryId, exists);
        return exists;
    }

--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java
@@ -30,6 +30,7 @@ public class QueryDataSetReq {
    private List<Filter> metricFilters = new ArrayList<>();
    private DateConf dateInfo;
    private Long limit = 2000L;
+    private Long offset = 0L;
    private QueryType queryType = QueryType.DETAIL;
    private boolean innerLayerNative = false;
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
@@ -3,11 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.Order;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -25,12 +21,7 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.schema.Table;
-import net.sf.jsqlparser.statement.select.GroupByElement;
-import net.sf.jsqlparser.statement.select.Limit;
-import net.sf.jsqlparser.statement.select.OrderByElement;
-import net.sf.jsqlparser.statement.select.ParenthesedSelect;
-import net.sf.jsqlparser.statement.select.PlainSelect;
-import net.sf.jsqlparser.statement.select.SelectItem;
+import net.sf.jsqlparser.statement.select.*;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -52,6 +43,7 @@ public class QueryStructReq extends SemanticQueryReq {
    private List<Filter> metricFilters = new ArrayList<>();
    private DateConf dateInfo;
    private long limit = Constants.DEFAULT_DETAIL_LIMIT;
+    private long offset;
    private QueryType queryType = QueryType.DETAIL;
    private boolean convertToSql = true;

@@ -170,12 +162,15 @@ public class QueryStructReq extends SemanticQueryReq {
        // 5. Set the limit clause
        plainSelect.setLimit(buildLimit(queryStructReq));

-        // 6. Set having clause
+        // 6. Set the offset clause
+        plainSelect.setOffset(buildOffset(queryStructReq));
+
+        // 7. Set the having clause
        plainSelect.setHaving(buildHavingClause(queryStructReq));

        select.setSelect(plainSelect);

-        // 6. Set where clause
+        // 8. Set the where clause
        return addWhereClauses(select.toString(), queryStructReq, isBizName);
    }

@@ -262,6 +257,15 @@ public class QueryStructReq extends SemanticQueryReq {
        return limit;
    }

+    private Offset buildOffset(QueryStructReq queryStructReq) {
+        if (Objects.isNull(queryStructReq.getOffset())) {
+            return null;
+        }
+        Offset offset = new Offset();
+        offset.setOffset(new LongValue(queryStructReq.getOffset()));
+        return offset;
+    }
+
    private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName)
            throws JSQLParserException {
        SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/RuleSqlCorrector.java
@@ -25,7 +25,7 @@ public class RuleSqlCorrector extends BaseSemanticCorrector {
    @Override
    public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
        ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
-        if(!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) {
+        if (!Boolean.parseBoolean(parserConfig.getParameterValue(PARSER_RULE_CORRECTOR_ENABLE))) {
            return;
        }

--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java
@@ -22,4 +22,9 @@ public abstract class MapResult implements Serializable {
        return this.getMapKey().equals(otherResult.getMapKey())
                && this.similarity < otherResult.similarity;
    }
+
+    public Boolean lessOrEqualSimilar(MapResult otherResult) {
+        return this.getMapKey().equals(otherResult.getMapKey())
+                && this.similarity <= otherResult.similarity;
+    }
 }
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java
@@ -75,6 +75,8 @@ public class MetaEmbeddingService {
            return dataSetIds.stream().map(dataSetId -> {
                Retrieval newRetrieval = new Retrieval();
                BeanUtils.copyProperties(retrieval, newRetrieval);
+                HashMap<String, Object> newMetadata = new HashMap<>(retrieval.getMetadata());
+                newRetrieval.setMetadata(newMetadata);
                newRetrieval.getMetadata().putIfAbsent("dataSetId",
                        dataSetId + Constants.UNDERLINE);
                return newRetrieval;
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
@@ -56,7 +56,7 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
        for (T oneRoundResult : oneRoundResults) {
            if (existResults.contains(oneRoundResult)) {
                boolean isDeleted = existResults.removeIf(existResult -> {
-                    boolean delete = existResult.lessSimilar(oneRoundResult);
+                    boolean delete = existResult.lessOrEqualSimilar(oneRoundResult);
                    if (delete) {
                        log.debug("deleted existResult:{}", existResult);
                    }
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java
@@ -59,7 +59,8 @@ public class ParserConfig extends ParameterConfig {

    @Override
    public List<Parameter> getSysParameters() {
-        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER,
-                PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD);
+        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE,
+                PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT,
+                PARSER_FIELDS_COUNT_THRESHOLD);
    }
 }
--- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java
+++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java
@@ -20,6 +20,7 @@ public class StructQuery {
    private List<Filter> metricFilters = new ArrayList();
    private DateConf dateInfo;
    private Long limit = 2000L;
+    private Long offset = 0L;
    private QueryType queryType;
    private List<Param> params = new ArrayList<>();
 }
--- a/webapp/packages/chat-sdk/src/components/ChatItem/index.tsx
+++ b/webapp/packages/chat-sdk/src/components/ChatItem/index.tsx
@@ -420,7 +420,7 @@ const ChatItem: React.FC<Props> = ({
    if (!!queryResults) {
      const exportData = queryResults.map(item => {
        return Object.keys(item).reduce((result, key) => {
-          const columnName = queryColumns?.find(column => column.nameEn === key)?.name || key;
+          const columnName = queryColumns?.find(column => column.name === key)?.name || key;
          result[columnName] = item[key];
          return result;
        }, {});
--- a/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx
+++ b/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx
@@ -48,12 +48,13 @@ const MetricTrend: React.FC<Props> = ({
  const { queryColumns, queryResults, aggregateInfo, entityInfo, chatContext } = data;
  const [chartType, setChartType] = useState('line');

-  const dateField: any = queryColumns?.find(
+  const dateField = queryColumns?.find(
    (column: any) => column.showType === 'DATE' || column.type === 'DATE'
  );
  const dateColumnName = dateField?.bizName || '';
-  const categoryColumnName =
-    queryColumns?.find((column: any) => column.showType === 'CATEGORY')?.bizName || '';
+  let categoryColumnName =
+      queryColumns?.find((column: any) => column.showType === 'CATEGORY')?.bizName || '';
+  categoryColumnName = categoryColumnName === dateColumnName ? '' : categoryColumnName;
  const metricFields = queryColumns?.filter((column: any) => column.showType === 'NUMBER');

  const currentMetricField = queryColumns?.find((column: any) => column.showType === 'NUMBER');
@@ -80,11 +81,11 @@ const MetricTrend: React.FC<Props> = ({
                <MetricInfo aggregateInfo={aggregateInfo} currentMetricField={currentMetricField} />
              )}
            <div className={`${prefixCls}-select-options`}>
-              <DateOptions
-                chatContext={chatContext}
-                currentDateOption={currentDateOption}
-                onSelectDateOption={onSelectDateOption}
-              />
+              {/*<DateOptions*/}
+              {/*  chatContext={chatContext}*/}
+              {/*  currentDateOption={currentDateOption}*/}
+              {/*  onSelectDateOption={onSelectDateOption}*/}
+              {/*/>*/}
              <div>
                <Select
                  defaultValue="line"
--- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx
+++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx
@@ -8,7 +8,7 @@ import { ISemantic } from '../../data';
 import {
  AGG_OPTIONS,
  DATE_FORMATTER,
-  DATE_OPTIONS,
+  // DATE_OPTIONS,
  DIM_OPTIONS,
  EnumDataSourceType,
  EnumModelDataType,
@@ -275,7 +275,7 @@ const ModelFieldForm: React.FC<Props> = ({
          }
        }
        if ([EnumDataSourceType.TIME, EnumDataSourceType.PARTITION_TIME].includes(type)) {
-          const { dateFormat, timeGranularity } = record;
+          const { dateFormat } = record;
          const dateFormatterOptions =
            type === EnumDataSourceType.PARTITION_TIME ? PARTITION_TIME_FORMATTER : DATE_FORMATTER;

@@ -302,25 +302,25 @@ const ModelFieldForm: React.FC<Props> = ({
                  <ExclamationCircleOutlined />
                </Tooltip>
              </Space>
-              <Space>
-                <span>时间粒度:</span>
-                <Select
-                  placeholder="时间粒度"
-                  value={timeGranularity === '' ? undefined : timeGranularity}
-                  onChange={(value) => {
-                    handleFieldChange(record, 'timeGranularity', value);
-                  }}
-                  defaultValue={timeGranularity === '' ? undefined : DATE_OPTIONS[0]}
-                  style={{ minWidth: 180 }}
-                  allowClear
-                >
-                  {DATE_OPTIONS.map((item) => (
-                    <Option key={item} value={item}>
-                      {item}
-                    </Option>
-                  ))}
-                </Select>
-              </Space>
+              {/*<Space>*/}
+              {/*  <span>时间粒度:</span>*/}
+              {/*  <Select*/}
+              {/*    placeholder="时间粒度"*/}
+              {/*    value={timeGranularity === '' ? undefined : timeGranularity}*/}
+              {/*    onChange={(value) => {*/}
+              {/*      handleFieldChange(record, 'timeGranularity', value);*/}
+              {/*    }}*/}
+              {/*    defaultValue={timeGranularity === '' ? undefined : DATE_OPTIONS[0]}*/}
+              {/*    style={{ minWidth: 180 }}*/}
+              {/*    allowClear*/}
+              {/*  >*/}
+              {/*    {DATE_OPTIONS.map((item) => (*/}
+              {/*      <Option key={item} value={item}>*/}
+              {/*        {item}*/}
+              {/*      </Option>*/}
+              {/*    ))}*/}
+              {/*  </Select>*/}
+              {/*</Space>*/}
            </Space>
          );
        }
Author	SHA1	Message	Date
beat4ocean	4b51dc6cca	Merge `f3ffcb94f1` into `d2a43a99c8`	2025-03-23 07:51:23 +00:00
beat4ocean	f3ffcb94f1	Merge branch 'tencentmusic:master' into master	2025-03-23 15:51:20 +08:00
jerryjzhang	d2a43a99c8	(feature)(headless)Support offset clause in struct query. Some checks failed supersonic CentOS CI / build (21) (push) Has been cancelled Details supersonic mac CI / build (21) (push) Has been cancelled Details supersonic ubuntu CI / build (21) (push) Has been cancelled Details supersonic windows CI / build (21) (push) Has been cancelled Details	2025-03-23 14:30:56 +08:00
iridescentpeo	db8f340e2d	修复同一模型被多个数据集引用时: (#2183 )	2025-03-23 14:14:08 +08:00
beat4ocean	de427b8093	Merge branch 'tencentmusic:master' into master	2025-03-19 12:23:39 +08:00
beat4ocean	82c188c17a	Merge branch 'tencentmusic:master' into master	2025-03-17 08:53:42 +08:00
beat4ocean	54e3e1487b	[fix][headless-fe] Property 'nameEn' does not exist on type 'ColumnType'.	2025-03-14 20:58:15 +08:00
beat4ocean	c2adb3314d	Merge branch 'tencentmusic:master' into master	2025-03-14 09:56:00 +08:00
beat4ocean	74f4732401	[fix][chat]Fix dashboard display issue when categoryColumnName and dateColumnName are identical. (cherry picked from commit `738093bc88`)	2025-03-13 13:57:19 +08:00
beat4ocean	2003c32c96	[Fix][Chat]Fix time granularity bug in model editing and dashboard display. (cherry picked from commit `b6d1525daa`)	2025-03-13 13:57:18 +08:00