Merge f3ffcb94f1 into d2a43a99c8

(cherry picked from commit 738093bc88)

common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java

@@ -49,7 +49,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
             try {
                 EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
                 Embedding embedding = embeddingModel.embed(question).content();
-                boolean existSegment = existSegment(collectionName,embeddingStore, query, embedding);
+                boolean existSegment =
+                        existSegment(collectionName, embeddingStore, query, embedding);
                 if (existSegment) {
                     continue;
                 }
@@ -62,8 +63,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
         }
     }
 
-    private boolean existSegment(String collectionName,EmbeddingStore embeddingStore, TextSegment query,
+    private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
-            Embedding embedding) {
+            TextSegment query, Embedding embedding) {
         String queryId = TextSegmentConvert.getQueryId(query);
         if (queryId == null) {
             return false;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java

@@ -30,6 +30,7 @@ public class QueryDataSetReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private Long limit = 2000L;
+    private Long offset = 0L;
     private QueryType queryType = QueryType.DETAIL;
     private boolean innerLayerNative = false;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -3,11 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.*;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -25,12 +21,7 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.schema.Table;
-import net.sf.jsqlparser.statement.select.GroupByElement;
+import net.sf.jsqlparser.statement.select.*;
-import net.sf.jsqlparser.statement.select.Limit;
-import net.sf.jsqlparser.statement.select.OrderByElement;
-import net.sf.jsqlparser.statement.select.ParenthesedSelect;
-import net.sf.jsqlparser.statement.select.PlainSelect;
-import net.sf.jsqlparser.statement.select.SelectItem;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -52,6 +43,7 @@ public class QueryStructReq extends SemanticQueryReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private long limit = Constants.DEFAULT_DETAIL_LIMIT;
+    private long offset;
     private QueryType queryType = QueryType.DETAIL;
     private boolean convertToSql = true;
 
@@ -170,12 +162,15 @@ public class QueryStructReq extends SemanticQueryReq {
         // 5. Set the limit clause
         plainSelect.setLimit(buildLimit(queryStructReq));
 
-        // 6. Set having clause
+        // 6. Set the offset clause
+        plainSelect.setOffset(buildOffset(queryStructReq));
+
+        // 7. Set the having clause
         plainSelect.setHaving(buildHavingClause(queryStructReq));
 
         select.setSelect(plainSelect);
 
-        // 6. Set where clause
+        // 8. Set the where clause
         return addWhereClauses(select.toString(), queryStructReq, isBizName);
     }
 
@@ -262,6 +257,15 @@ public class QueryStructReq extends SemanticQueryReq {
         return limit;
     }
 
+    private Offset buildOffset(QueryStructReq queryStructReq) {
+        if (Objects.isNull(queryStructReq.getOffset())) {
+            return null;
+        }
+        Offset offset = new Offset();
+        offset.setOffset(new LongValue(queryStructReq.getOffset()));
+        return offset;
+    }
+
     private String addWhereClauses(String sql, QueryStructReq queryStructReq, boolean isBizName)
             throws JSQLParserException {
         SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MapResult.java

@@ -22,4 +22,9 @@ public abstract class MapResult implements Serializable {
         return this.getMapKey().equals(otherResult.getMapKey())
                 && this.similarity < otherResult.similarity;
     }
+
+    public Boolean lessOrEqualSimilar(MapResult otherResult) {
+        return this.getMapKey().equals(otherResult.getMapKey())
+                && this.similarity <= otherResult.similarity;
+    }
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MetaEmbeddingService.java

@@ -75,6 +75,8 @@ public class MetaEmbeddingService {
             return dataSetIds.stream().map(dataSetId -> {
                 Retrieval newRetrieval = new Retrieval();
                 BeanUtils.copyProperties(retrieval, newRetrieval);
+                HashMap<String, Object> newMetadata = new HashMap<>(retrieval.getMetadata());
+                newRetrieval.setMetadata(newMetadata);
                 newRetrieval.getMetadata().putIfAbsent("dataSetId",
                         dataSetId + Constants.UNDERLINE);
                 return newRetrieval;

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java

@@ -56,7 +56,7 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
         for (T oneRoundResult : oneRoundResults) {
             if (existResults.contains(oneRoundResult)) {
                 boolean isDeleted = existResults.removeIf(existResult -> {
-                    boolean delete = existResult.lessSimilar(oneRoundResult);
+                    boolean delete = existResult.lessOrEqualSimilar(oneRoundResult);
                     if (delete) {
                         log.debug("deleted existResult:{}", existResult);
                     }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java

@@ -59,7 +59,8 @@ public class ParserConfig extends ParameterConfig {
 
     @Override
     public List<Parameter> getSysParameters() {
-        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE, PARSER_FEW_SHOT_NUMBER,
+        return Lists.newArrayList(PARSER_LINKING_VALUE_ENABLE, PARSER_RULE_CORRECTOR_ENABLE,
-                PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT, PARSER_FIELDS_COUNT_THRESHOLD);
+                PARSER_FEW_SHOT_NUMBER, PARSER_SELF_CONSISTENCY_NUMBER, PARSER_SHOW_COUNT,
+                PARSER_FIELDS_COUNT_THRESHOLD);
     }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java

@@ -20,6 +20,7 @@ public class StructQuery {
     private List<Filter> metricFilters = new ArrayList();
     private DateConf dateInfo;
     private Long limit = 2000L;
+    private Long offset = 0L;
     private QueryType queryType;
     private List<Param> params = new ArrayList<>();
 }

webapp/packages/chat-sdk/src/components/ChatItem/index.tsx

@@ -420,7 +420,7 @@ const ChatItem: React.FC<Props> = ({
     if (!!queryResults) {
       const exportData = queryResults.map(item => {
         return Object.keys(item).reduce((result, key) => {
-          const columnName = queryColumns?.find(column => column.nameEn === key)?.name || key;
+          const columnName = queryColumns?.find(column => column.name === key)?.name || key;
           result[columnName] = item[key];
           return result;
         }, {});

webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx

@@ -48,12 +48,13 @@ const MetricTrend: React.FC<Props> = ({
   const { queryColumns, queryResults, aggregateInfo, entityInfo, chatContext } = data;
   const [chartType, setChartType] = useState('line');
 
-  const dateField: any = queryColumns?.find(
+  const dateField = queryColumns?.find(
     (column: any) => column.showType === 'DATE' || column.type === 'DATE'
   );
   const dateColumnName = dateField?.bizName || '';
-  const categoryColumnName =
+  let categoryColumnName =
       queryColumns?.find((column: any) => column.showType === 'CATEGORY')?.bizName || '';
+  categoryColumnName = categoryColumnName === dateColumnName ? '' : categoryColumnName;
   const metricFields = queryColumns?.filter((column: any) => column.showType === 'NUMBER');
 
   const currentMetricField = queryColumns?.find((column: any) => column.showType === 'NUMBER');
@@ -80,11 +81,11 @@ const MetricTrend: React.FC<Props> = ({
                 <MetricInfo aggregateInfo={aggregateInfo} currentMetricField={currentMetricField} />
               )}
             <div className={`${prefixCls}-select-options`}>
-              <DateOptions
+              {/*<DateOptions*/}
-                chatContext={chatContext}
+              {/*  chatContext={chatContext}*/}
-                currentDateOption={currentDateOption}
+              {/*  currentDateOption={currentDateOption}*/}
-                onSelectDateOption={onSelectDateOption}
+              {/*  onSelectDateOption={onSelectDateOption}*/}
-              />
+              {/*/>*/}
               <div>
                 <Select
                   defaultValue="line"

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelFieldForm.tsx

@@ -8,7 +8,7 @@ import { ISemantic } from '../../data';
 import {
   AGG_OPTIONS,
   DATE_FORMATTER,
-  DATE_OPTIONS,
+  // DATE_OPTIONS,
   DIM_OPTIONS,
   EnumDataSourceType,
   EnumModelDataType,
@@ -275,7 +275,7 @@ const ModelFieldForm: React.FC<Props> = ({
           }
         }
         if ([EnumDataSourceType.TIME, EnumDataSourceType.PARTITION_TIME].includes(type)) {
-          const { dateFormat, timeGranularity } = record;
+          const { dateFormat } = record;
           const dateFormatterOptions =
             type === EnumDataSourceType.PARTITION_TIME ? PARTITION_TIME_FORMATTER : DATE_FORMATTER;
 
@@ -302,25 +302,25 @@ const ModelFieldForm: React.FC<Props> = ({
                   <ExclamationCircleOutlined />
                 </Tooltip>
               </Space>
-              <Space>
+              {/*<Space>*/}
-                <span>时间粒度:</span>
+              {/*  <span>时间粒度:</span>*/}
-                <Select
+              {/*  <Select*/}
-                  placeholder="时间粒度"
+              {/*    placeholder="时间粒度"*/}
-                  value={timeGranularity === '' ? undefined : timeGranularity}
+              {/*    value={timeGranularity === '' ? undefined : timeGranularity}*/}
-                  onChange={(value) => {
+              {/*    onChange={(value) => {*/}
-                    handleFieldChange(record, 'timeGranularity', value);
+              {/*      handleFieldChange(record, 'timeGranularity', value);*/}
-                  }}
+              {/*    }}*/}
-                  defaultValue={timeGranularity === '' ? undefined : DATE_OPTIONS[0]}
+              {/*    defaultValue={timeGranularity === '' ? undefined : DATE_OPTIONS[0]}*/}
-                  style={{ minWidth: 180 }}
+              {/*    style={{ minWidth: 180 }}*/}
-                  allowClear
+              {/*    allowClear*/}
-                >
+              {/*  >*/}
-                  {DATE_OPTIONS.map((item) => (
+              {/*    {DATE_OPTIONS.map((item) => (*/}
-                    <Option key={item} value={item}>
+              {/*      <Option key={item} value={item}>*/}
-                      {item}
+              {/*        {item}*/}
-                    </Option>
+              {/*      </Option>*/}
-                  ))}
+              {/*    ))}*/}
-                </Select>
+              {/*  </Select>*/}
-              </Space>
+              {/*</Space>*/}
             </Space>
           );
         }