Merge 5b45cfbad7 into e55f43c737

* [improvement][launcher]Use API to get element ID avoiding hard-code.

* [fix][launcher]Fix mysql scripts.

* [improvement][launcher]Support DuckDB database and refactor translator code structure.

* [improvement][headless-fe] Revamped the interaction for semantic modeling routing and successfully implemented the switching between dimension and dataset management.

* [improvement][Headless] Add table ddl in Dbschema

* [improvement][Headless] Add get database by type

* [improvement][Headless] Supports automatic batch creation of models based on db table names.

* [improvement][Headless] Supports getting domain by bizName

* [improvement][launcher]Refactor unit tests and demo data.

* [fix][launcher]Change default vector dimension to 512.

* [improvement](Dict) add dimValueAliasMap info for KnowledgeBaseService

* [improvement][headless]Use QueryWrapper to replace hard-code SQL in mapper xml.

* [improvement][chat]Introduce ChatMemory to delegate ChatMemoryDO.

* [fix][common]Fix embedding store sys configs.

* [fix][common]Fix postgres schema, using varchar instead of char.

* [improvement][launcher]Change supersonic docker deployment from mysql to postgres.

* [Fix][launcher]Fix a number of issues related to semantic modeling.

* [Fix][headless]Fix the evaluation logic of agg type.

* [fix][assembly]Fix Dockerfile and add docker compose run script.

* [fix][chat]Fix "multiple assignments to same column "similar_queries".

* [improvement][headless]Use LamdaQueryWrapper to avoid hard-coded column names.

* [improvement][headless]Refactor headless infra to support advanced semantic modelling.

* [improvement][headless]Change class name `Dim` to `Dimension`.

* [improvement][chat]Introduce `TimeFieldMapper` to always map time field.

* [fix][headless]Remove unnecessary dimension existence check.

* [fix][chat]Fix adjusted filters don't take effect.

---------

.github/workflows/docker-publish.yml

@@ -0,0 +1,34 @@
+name: Docker Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version of the Docker image'
+        required: true
+        default: 'latest'
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and publish Docker image
+        run: |
+          VERSION=${{ github.event.inputs.version }}
+          chmod +x docker/docker-build.sh
+          chmod +x docker/docker-publish.sh
+          sh docker/docker-build.sh $VERSION
+          sh docker/docker-publish.sh $VERSION

CHANGELOG.md

@@ -7,11 +7,11 @@
 ## SuperSonic [0.9.8] - 2024-11-01
 - Add LLM management module to reuse connection across agents.
 - Add ChatAPP configuration sub-module in Agent Management.
-- Add dimension value management sub-module.
+- Enhance dimension value management sub-module.
 - Enhance memory management and term management sub-module.
-- Support semantic translation of complex S2SQL.
+- Enhance semantic translation of complex S2SQL.
+- Enhance user experience in Chat UI.
 - Introduce LLM-based semantic corrector and data interpreter.
-- Introduce new experience in Chat UI.
 
 ## SuperSonic [0.9.2] - 2024-06-01
 

README.md

@@ -4,7 +4,7 @@
 
 # SuperSonic
 
-SuperSonic is the next-generation BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other:
+SuperSonic is the next-generation AI+BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other:
 
 - Chat BI's Text2SQL gets augmented with context-retrieval from semantic models.
 - Headless BI's query interface gets extended with natural language API.

assembly/bin/supersonic-build.sh

@@ -4,7 +4,13 @@ sbinDir=$(cd "$(dirname "$0")"; pwd)
 chmod +x $sbinDir/supersonic-common.sh
 source $sbinDir/supersonic-common.sh
 cd $projectDir
-MVN_VERSION=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[^\[]')
+
+MVN_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout | grep -v '^\[' | sed -n '/^[0-9]/p')
+if [ -z "$MVN_VERSION" ]; then
+  echo "Failed to retrieve Maven project version."
+  exit 1
+fi
+echo "Maven project version: $MVN_VERSION"
 
 cd $baseDir
 service=$1

auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/UserWithPassword.java

@@ -11,6 +11,7 @@ import java.util.Map;
 import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_CREATE_TIME;
 import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_IS_ADMIN;
 import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_DISPLAY_NAME;
+import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_EMAIL;
 import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_ID;
 import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_NAME;
 import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_PASSWORD;
@@ -38,6 +39,7 @@ public class UserWithPassword extends User {
         claims.put(TOKEN_USER_NAME, StringUtils.isEmpty(user.getName()) ? "" : user.getName());
         claims.put(TOKEN_USER_PASSWORD,
                 StringUtils.isEmpty(user.getPassword()) ? "" : user.getPassword());
+        claims.put(TOKEN_USER_EMAIL, StringUtils.isEmpty(user.getEmail()) ? "" : user.getEmail());
         claims.put(TOKEN_USER_DISPLAY_NAME, user.getDisplayName());
         claims.put(TOKEN_CREATE_TIME, System.currentTimeMillis());
         claims.put(TOKEN_IS_ADMIN, user.getIsAdmin());

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java

@@ -51,7 +51,7 @@ public class UserRepositoryImpl implements UserRepository {
     @Override
     public List<UserTokenDO> getUserTokenListByName(String userName) {
         QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("user_name", userName);
+        queryWrapper.lambda().eq(UserTokenDO::getUserName, userName);
         return userTokenDOMapper.selectList(queryWrapper);
     }
 
@@ -68,7 +68,7 @@ public class UserRepositoryImpl implements UserRepository {
     @Override
     public void deleteUserTokenByName(String userName) {
         QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("user_name", userName);
+        queryWrapper.lambda().eq(UserTokenDO::getUserName, userName);
         userTokenDOMapper.delete(queryWrapper);
     }
 

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java

@@ -4,9 +4,11 @@ import javax.validation.constraints.NotNull;
 
 import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
 import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
+import lombok.Builder;
 import lombok.Data;
 
 @Data
+@Builder
 public class ChatMemoryUpdateReq {
 
     @NotNull(message = "id不可为空")

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java

@@ -1,22 +0,0 @@
-package com.tencent.supersonic.chat.api.pojo.request;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.ToString;
-
-import java.util.List;
-
-/** the entity info about the model */
-@Data
-@AllArgsConstructor
-@ToString
-@NoArgsConstructor
-public class Entity {
-
-    /** uniquely identifies an entity */
-    private Long entityId;
-
-    /** entity name list */
-    private List<String> names;
-}

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java

@@ -1,14 +0,0 @@
-package com.tencent.supersonic.chat.api.pojo.response;
-
-import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import lombok.Data;
-
-import java.util.List;
-
-@Data
-public class EntityRichInfoResp {
-    /** entity alias */
-    private List<String> names;
-
-    private SchemaElement dimItem;
-}

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java

@@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.api.pojo.response;
 import com.tencent.supersonic.common.pojo.QueryAuthorization;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.headless.api.pojo.AggregateInfo;
-import com.tencent.supersonic.headless.api.pojo.EntityInfo;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.response.QueryState;
@@ -26,7 +25,6 @@ public class QueryResult {
     private String textResult;
     private String textSummary;
     private Long queryTimeCost;
-    private EntityInfo entityInfo;
     private List<SchemaElement> recommendedDimensions;
     private AggregateInfo aggregateInfo;
     private String errorMsg;

chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java

@@ -1,9 +1,11 @@
 package com.tencent.supersonic.chat.server.agent;
 
 import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
 import com.tencent.supersonic.chat.server.memory.MemoryReviewTask;
 import com.tencent.supersonic.common.pojo.ChatApp;
 import com.tencent.supersonic.common.pojo.RecordInfo;
+import com.tencent.supersonic.common.pojo.User;
 import lombok.Data;
 import org.springframework.util.CollectionUtils;
 
@@ -12,6 +14,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @Data
@@ -33,6 +36,8 @@ public class Agent extends RecordInfo {
     private String toolConfig;
     private Map<String, ChatApp> chatAppConfig = Collections.emptyMap();
     private VisualConfig visualConfig;
+    private List<String> admins = Lists.newArrayList();
+    private List<String> viewers = Lists.newArrayList();
 
     public List<String> getTools(AgentToolType type) {
         Map<String, Object> map = JSONObject.parseObject(toolConfig, Map.class);
@@ -105,4 +110,9 @@ public class Agent extends RecordInfo {
                 .filter(dataSetIds -> !CollectionUtils.isEmpty(dataSetIds))
                 .flatMap(Collection::stream).collect(Collectors.toSet());
     }
+
+    public boolean contains(User user, Function<Agent, List<String>> list) {
+        return list.apply(this).contains(user.getName());
+    }
+
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/PlainTextExecutor.java

@@ -26,7 +26,7 @@ import java.util.stream.Collectors;
 public class PlainTextExecutor implements ChatQueryExecutor {
 
     public static final String APP_KEY = "SMALL_TALK";
-    private static final String INSTRUCTION = "" + "#Role: You are a nice person to talk to."
+    private static final String INSTRUCTION = "#Role: You are a nice person to talk to."
             + "\n#Task: Respond quickly and nicely to the user."
             + "\n#Rules: 1.ALWAYS use the same language as the `#Current Input`."
             + "\n#History Inputs: %s" + "\n#Current Input: %s" + "\n#Response: ";

chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.executor;
 
 import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
-import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
 import com.tencent.supersonic.chat.server.pojo.ChatContext;
+import com.tencent.supersonic.chat.server.pojo.ChatMemory;
 import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
 import com.tencent.supersonic.chat.server.service.ChatContextService;
 import com.tencent.supersonic.chat.server.service.MemoryService;
@@ -44,7 +44,7 @@ public class SqlExecutor implements ChatQueryExecutor {
                                 Text2SQLExemplar.class);
 
                 MemoryService memoryService = ContextUtils.getBean(MemoryService.class);
-                memoryService.createMemory(ChatMemoryDO.builder()
+                memoryService.createMemory(ChatMemory.builder()
                         .agentId(executeContext.getAgent().getId()).status(MemoryStatus.PENDING)
                         .question(exemplar.getQuestion()).sideInfo(exemplar.getSideInfo())
                         .dbSchema(exemplar.getDbSchema()).s2sql(exemplar.getSql())

chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java

@@ -1,8 +1,10 @@
 package com.tencent.supersonic.chat.server.memory;
 
 import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
+import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
+import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
 import com.tencent.supersonic.chat.server.agent.Agent;
-import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
+import com.tencent.supersonic.chat.server.pojo.ChatMemory;
 import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.chat.server.service.MemoryService;
 import com.tencent.supersonic.common.pojo.ChatApp;
@@ -21,6 +23,7 @@ import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -57,24 +60,36 @@ public class MemoryReviewTask {
 
     @Scheduled(fixedDelay = 60 * 1000)
     public void review() {
-        memoryService.getMemoriesForLlmReview().stream().forEach(memory -> {
+        List<Agent> agentList = agentService.getAgents();
+        for (Agent agent : agentList) {
+            if (!agent.enableMemoryReview()) {
+                continue;
+            }
+            ChatMemoryFilter chatMemoryFilter =
+                    ChatMemoryFilter.builder().agentId(agent.getId()).build();
+            memoryService.getMemories(chatMemoryFilter).forEach(memory -> {
                 try {
-                processMemory(memory);
+                    processMemory(memory, agent);
                 } catch (Exception e) {
                     log.error("Exception occurred while processing memory with id {}: {}",
                             memory.getId(), e.getMessage(), e);
                 }
             });
         }
+    }
 
-    private void processMemory(ChatMemoryDO m) {
+    private void processMemory(ChatMemory m, Agent agent) {
-        Agent chatAgent = agentService.getAgent(m.getAgentId());
+        if (Objects.isNull(agent)) {
-        if (Objects.isNull(chatAgent)) {
             log.warn("Agent id {} not found or memory review disabled", m.getAgentId());
             return;
         }
 
-        ChatApp chatApp = chatAgent.getChatAppConfig().get(APP_KEY);
+        // if either LLM or human has reviewed, just return
+        if (Objects.nonNull(m.getLlmReviewRet()) || Objects.nonNull(m.getHumanReviewRet())) {
+            return;
+        }
+
+        ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
         if (Objects.isNull(chatApp) || !chatApp.isEnable()) {
             return;
         }
@@ -90,23 +105,23 @@ public class MemoryReviewTask {
                     response);
             processResponse(response, m);
         } else {
-            log.debug("ChatLanguageModel not found for agent:{}", chatAgent.getId());
+            log.debug("ChatLanguageModel not found for agent:{}", agent.getId());
         }
     }
 
-    private String createPromptString(ChatMemoryDO m, String promptTemplate) {
+    private String createPromptString(ChatMemory m, String promptTemplate) {
         return String.format(promptTemplate, m.getQuestion(), m.getDbSchema(), m.getSideInfo(),
                 m.getS2sql());
     }
 
-    private void processResponse(String response, ChatMemoryDO m) {
+    private void processResponse(String response, ChatMemory m) {
         Matcher matcher = OUTPUT_PATTERN.matcher(response);
         if (matcher.find()) {
             m.setLlmReviewRet(MemoryReviewResult.getMemoryReviewResult(matcher.group(1)));
             m.setLlmReviewCmt(matcher.group(2));
             // directly enable memory if the LLM determines it positive
             if (MemoryReviewResult.POSITIVE.equals(m.getLlmReviewRet())) {
-                memoryService.enableMemory(m);
+                m.setStatus(MemoryStatus.ENABLED);
             }
             memoryService.updateMemory(m);
         }

chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java

@@ -83,11 +83,15 @@ public class NL2SQLParser implements ChatQueryParser {
         if (Objects.isNull(parseContext.getRequest().getSelectedParse())) {
             QueryNLReq queryNLReq = QueryReqConverter.buildQueryNLReq(parseContext);
             queryNLReq.setText2SQLType(Text2SQLType.ONLY_RULE);
+            if (parseContext.enableLLM()) {
+                queryNLReq.setText2SQLType(Text2SQLType.NONE);
+            }
 
             // for every requested dataSet, recursively invoke rule-based parser with different
             // mapModes
             Set<Long> requestedDatasets = queryNLReq.getDataSetIds();
             List<SemanticParseInfo> candidateParses = Lists.newArrayList();
+            StringBuilder errMsg = new StringBuilder();
             for (Long datasetId : requestedDatasets) {
                 queryNLReq.setDataSetIds(Collections.singleton(datasetId));
                 ChatParseResp parseResp = new ChatParseResp(parseContext.getRequest().getQueryId());
@@ -100,6 +104,10 @@ public class NL2SQLParser implements ChatQueryParser {
                     queryNLReq.setMapModeEnum(MapModeEnum.LOOSE);
                     doParse(queryNLReq, parseResp);
                 }
+                if (parseResp.getSelectedParses().isEmpty()) {
+                    errMsg.append(parseResp.getErrorMsg());
+                    continue;
+                }
                 // for one dataset select the top 1 parse after sorting
                 SemanticParseInfo.sort(parseResp.getSelectedParses());
                 candidateParses.add(parseResp.getSelectedParses().get(0));
@@ -110,6 +118,10 @@ public class NL2SQLParser implements ChatQueryParser {
             SemanticParseInfo.sort(candidateParses);
             parseContext.getResponse().setSelectedParses(
                     candidateParses.subList(0, Math.min(parserShowCount, candidateParses.size())));
+            if (parseContext.getResponse().getSelectedParses().isEmpty()) {
+                parseContext.getResponse().setState(ParseResp.ParseState.FAILED);
+                parseContext.getResponse().setErrorMsg(errMsg.toString());
+            }
         }
 
         // next go with llm-based parsers unless LLM is disabled or use feedback is needed.

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java

@@ -40,4 +40,8 @@ public class AgentDO {
     private String chatModelConfig;
 
     private String visualConfig;
+
+    private String admin;
+
+    private String viewer;
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatConfigDO.java

@@ -1,5 +1,8 @@
 package com.tencent.supersonic.chat.server.persistence.dataobject;
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
 import lombok.ToString;
 
@@ -7,9 +10,10 @@ import java.util.Date;
 
 @Data
 @ToString
+@TableName("s2_chat_config")
 public class ChatConfigDO {
 
-    /** database auto-increment primary key */
+    @TableId(type = IdType.AUTO)
     private Long id;
 
     private Long modelId;

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatContextDO.java

@@ -1,15 +1,21 @@
 package com.tencent.supersonic.chat.server.persistence.dataobject;
 
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
 
 import java.io.Serializable;
 import java.time.Instant;
 
 @Data
+@TableName("s2_chat_context")
 public class ChatContextDO implements Serializable {
 
+    @TableId
     private Integer chatId;
     private Instant modifiedAt;
+    @TableField("query_user")
     private String user;
     private String queryText;
     private String semanticParse;

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java

@@ -4,17 +4,17 @@ import com.baomidou.mybatisplus.annotation.IdType;
 import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
-import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
+import lombok.AllArgsConstructor;
-import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
 import lombok.Builder;
 import lombok.Data;
-import lombok.ToString;
+import lombok.NoArgsConstructor;
 
 import java.util.Date;
 
 @Data
 @Builder
-@ToString
+@NoArgsConstructor
+@AllArgsConstructor
 @TableName("s2_chat_memory")
 public class ChatMemoryDO {
     @TableId(type = IdType.AUTO)
@@ -36,16 +36,16 @@ public class ChatMemoryDO {
     private String s2sql;
 
     @TableField("status")
-    private MemoryStatus status;
+    private String status;
 
     @TableField("llm_review")
-    private MemoryReviewResult llmReviewRet;
+    private String llmReviewRet;
 
     @TableField("llm_comment")
     private String llmReviewCmt;
 
     @TableField("human_review")
-    private MemoryReviewResult humanReviewRet;
+    private String humanReviewRet;
 
     @TableField("human_comment")
     private String humanReviewCmt;

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/QueryDO.java

@@ -5,9 +5,6 @@ import lombok.Data;
 @Data
 public class QueryDO {
 
-    public String aggregator = "trend";
-    public String startTime;
-    public String endTime;
     private long id;
     private long questionId;
     private String createTime;
@@ -25,7 +22,6 @@ public class QueryDO {
     private int topNum;
     private String querySql;
     private Object queryColumn;
-    private Object entityInfo;
     private int score;
     private String feedback;
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatConfigMapper.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.chat.server.persistence.mapper;
 
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO;
 import org.apache.ibatis.annotations.Mapper;
@@ -7,11 +8,7 @@ import org.apache.ibatis.annotations.Mapper;
 import java.util.List;
 
 @Mapper
-public interface ChatConfigMapper {
+public interface ChatConfigMapper extends BaseMapper<ChatConfigDO> {
-
-    Long addConfig(ChatConfigDO chaConfigPO);
-
-    Long editConfig(ChatConfigDO chaConfigPO);
 
     List<ChatConfigDO> search(ChatConfigFilterInternal filterInternal);
 

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatContextMapper.java

@@ -1,14 +1,11 @@
 package com.tencent.supersonic.chat.server.persistence.mapper;
 
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO;
 import org.apache.ibatis.annotations.Mapper;
 
 @Mapper
-public interface ChatContextMapper {
+public interface ChatContextMapper extends BaseMapper<ChatContextDO> {
 
     ChatContextDO getContextByChatId(Integer chatId);
-
-    int updateContext(ChatContextDO contextDO);
-
-    int addContext(ChatContextDO contextDO);
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatConfigRepositoryImpl.java

@@ -32,15 +32,15 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
     @Override
     public Long createConfig(ChatConfig chaConfig) {
         ChatConfigDO chaConfigDO = chatConfigHelper.chatConfig2DO(chaConfig);
-        chatConfigMapper.addConfig(chaConfigDO);
+        chatConfigMapper.insert(chaConfigDO);
         return chaConfigDO.getId();
     }
 
     @Override
     public Long updateConfig(ChatConfig chaConfig) {
         ChatConfigDO chaConfigDO = chatConfigHelper.chatConfig2DO(chaConfig);
-
+        chatConfigMapper.updateById(chaConfigDO);
-        return chatConfigMapper.editConfig(chaConfigDO);
+        return chaConfigDO.getId();
     }
 
     @Override

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java

@@ -35,12 +35,7 @@ public class ChatContextRepositoryImpl implements ChatContextRepository {
 
     @Override
     public void updateContext(ChatContext chatCtx) {
-        ChatContextDO context = cast(chatCtx);
+        chatContextMapper.insertOrUpdate(cast(chatCtx));
-        if (chatContextMapper.getContextByChatId(chatCtx.getChatId()) == null) {
-            chatContextMapper.addContext(context);
-        } else {
-            chatContextMapper.updateContext(context);
-        }
     }
 
     private ChatContext cast(ChatContextDO contextDO) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java

@@ -20,7 +20,6 @@ import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryReposi
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.PageUtils;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;

chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginQueryManager.java

@@ -7,7 +7,7 @@ import java.util.Map;
 
 public class PluginQueryManager {
 
-    private static Map<String, PluginSemanticQuery> pluginQueries = new HashMap<>();
+    private static final Map<String, PluginSemanticQuery> pluginQueries = new HashMap<>();
 
     public static void register(String queryMode, PluginSemanticQuery pluginSemanticQuery) {
         pluginQueries.put(queryMode, pluginSemanticQuery);

chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatMemory.java

@@ -0,0 +1,48 @@
+package com.tencent.supersonic.chat.server.pojo;
+
+import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
+import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+import java.util.Date;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@ToString
+public class ChatMemory {
+    private Long id;
+
+    private Integer agentId;
+
+    private String question;
+
+    private String sideInfo;
+
+    private String dbSchema;
+
+    private String s2sql;
+
+    private MemoryStatus status;
+
+    private MemoryReviewResult llmReviewRet;
+
+    private String llmReviewCmt;
+
+    private MemoryReviewResult humanReviewRet;
+
+    private String humanReviewCmt;
+
+    private String createdBy;
+
+    private Date createdAt;
+
+    private String updatedBy;
+
+    private Date updatedAt;
+}

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioCalcProcessor.java

@@ -117,8 +117,12 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
 
             CompletableFuture.allOf(metricInfoRoll, metricInfoOver).join();
 
+            if (metricInfoRoll.get().getName() != null) {
                 metricInfo.setName(metricInfoRoll.get().getName());
+            }
+            if (metricInfoOver.get().getValue() != null) {
                 metricInfo.setValue(metricInfoRoll.get().getValue());
+            }
             metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics());
             metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics());
 

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java

@@ -39,7 +39,8 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor {
     }
 
     private void fillSimilarMetric(SemanticParseInfo parseInfo) {
-        if (!parseInfo.getQueryType().equals(QueryType.AGGREGATE)
+        if (Objects.isNull(parseInfo.getQueryType())
+                || !parseInfo.getQueryType().equals(QueryType.AGGREGATE)
                 || parseInfo.getMetrics().size() > METRIC_RECOMMEND_SIZE
                 || CollectionUtils.isEmpty(parseInfo.getMetrics())) {
             return;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java

@@ -1,13 +1,30 @@
 package com.tencent.supersonic.chat.server.processor.parse;
 
-import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
+import com.google.common.collect.Lists;
 import com.tencent.supersonic.chat.server.pojo.ParseContext;
+import com.tencent.supersonic.common.jsqlparser.FieldExpression;
+import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
+import com.tencent.supersonic.common.pojo.enums.QueryType;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
+import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.headless.api.pojo.SqlInfo;
 import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
+import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -15,20 +32,21 @@ import java.util.stream.Collectors;
 /**
  * ParseInfoFormatProcessor formats parse info to make it more readable to the users.
  **/
+@Slf4j
 public class ParseInfoFormatProcessor implements ParseResultProcessor {
     @Override
     public void process(ParseContext parseContext) {
         parseContext.getResponse().getSelectedParses().forEach(p -> {
-            if (PluginQueryManager.isPluginQuery(p.getQueryMode())
+            if (Objects.isNull(p.getDataSet()) || Objects.isNull(p.getSqlInfo().getParsedS2SQL())) {
-                    || "PLAIN_TEXT".equals(p.getQueryMode())) {
                 return;
             }
 
-            formatNL2SQLParseInfo(p);
+            buildParseInfoFromSQL(p);
+            buildTextInfo(p);
         });
     }
 
-    private static void formatNL2SQLParseInfo(SemanticParseInfo parseInfo) {
+    private void buildTextInfo(SemanticParseInfo parseInfo) {
         StringBuilder textBuilder = new StringBuilder();
         textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
         List<String> metricNames = parseInfo.getMetrics().stream().map(SchemaElement::getName)
@@ -60,4 +78,198 @@ public class ParseInfoFormatProcessor implements ParseResultProcessor {
         }
         parseInfo.setTextInfo(textBuilder.toString());
     }
+
+
+    private void buildParseInfoFromSQL(SemanticParseInfo parseInfo) {
+        SqlInfo sqlInfo = parseInfo.getSqlInfo();
+        String s2SQL = sqlInfo.getCorrectedS2SQL();
+        if (StringUtils.isBlank(s2SQL)) {
+            return;
+        }
+
+        parseQueryType(parseInfo);
+        List<FieldExpression> expressions = SqlSelectHelper.getFilterExpression(s2SQL);
+        Long dataSetId = parseInfo.getDataSetId();
+        SemanticLayerService semanticLayerService =
+                ContextUtils.getBean(SemanticLayerService.class);
+        DataSetSchema dsSchema = semanticLayerService.getDataSetSchema(dataSetId);
+
+        // extract date filter from S2SQL
+        try {
+            if (parseInfo.getDateInfo() == null && !CollectionUtils.isEmpty(expressions)) {
+                parseInfo.setDateInfo(extractDateFilter(expressions, dsSchema));
+            }
+        } catch (Exception e) {
+            log.error("failed to extract date range:", e);
+        }
+
+        // extract dimension filters from S2SQL
+        try {
+            List<QueryFilter> queryFilters = extractDimensionFilter(dsSchema, expressions);
+            parseInfo.getDimensionFilters().addAll(queryFilters);
+        } catch (Exception e) {
+            log.error("failed to extract dimension filters:", e);
+        }
+
+        // extract metrics from S2SQL
+        List<String> allFields =
+                filterDateField(dsSchema, SqlSelectHelper.getAllSelectFields(s2SQL));
+        Set<SchemaElement> metrics = matchSchemaElements(allFields, dsSchema.getMetrics());
+        parseInfo.setMetrics(metrics);
+
+        // extract dimensions from S2SQL
+        if (QueryType.AGGREGATE.equals(parseInfo.getQueryType())) {
+            List<String> groupByFields = SqlSelectHelper.getGroupByFields(s2SQL);
+            List<String> groupByDimensions = filterDateField(dsSchema, groupByFields);
+            parseInfo.setDimensions(
+                    matchSchemaElements(groupByDimensions, dsSchema.getDimensions()));
+        } else if (QueryType.DETAIL.equals(parseInfo.getQueryType())) {
+            List<String> selectFields = SqlSelectHelper.getSelectFields(s2SQL);
+            List<String> selectDimensions = filterDateField(dsSchema, selectFields);
+            parseInfo
+                    .setDimensions(matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
+        }
+    }
+
+    private Set<SchemaElement> matchSchemaElements(List<String> allFields,
+            Set<SchemaElement> elements) {
+        return elements.stream().filter(schemaElement -> {
+            if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
+                return allFields.contains(schemaElement.getName());
+            }
+            Set<String> allFieldsSet = new HashSet<>(allFields);
+            Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
+            List<String> intersection =
+                    allFieldsSet.stream().filter(aliasSet::contains).collect(Collectors.toList());
+            return allFields.contains(schemaElement.getName())
+                    || !CollectionUtils.isEmpty(intersection);
+        }).collect(Collectors.toSet());
+    }
+
+    private List<String> filterDateField(DataSetSchema dataSetSchema, List<String> allFields) {
+        return allFields.stream().filter(entry -> !isPartitionDimension(dataSetSchema, entry))
+                .collect(Collectors.toList());
+    }
+
+    private List<QueryFilter> extractDimensionFilter(DataSetSchema dsSchema,
+            List<FieldExpression> fieldExpressions) {
+
+        Map<String, SchemaElement> fieldNameToElement = getNameToElement(dsSchema);
+        List<QueryFilter> result = Lists.newArrayList();
+        for (FieldExpression expression : fieldExpressions) {
+            QueryFilter dimensionFilter = new QueryFilter();
+            dimensionFilter.setValue(expression.getFieldValue());
+            SchemaElement schemaElement = fieldNameToElement.get(expression.getFieldName());
+            if (Objects.isNull(schemaElement)
+                    || isPartitionDimension(dsSchema, schemaElement.getName())) {
+                continue;
+            }
+            dimensionFilter.setName(schemaElement.getName());
+            dimensionFilter.setBizName(schemaElement.getBizName());
+            dimensionFilter.setElementID(schemaElement.getId());
+
+            FilterOperatorEnum operatorEnum =
+                    FilterOperatorEnum.getSqlOperator(expression.getOperator());
+            dimensionFilter.setOperator(operatorEnum);
+            dimensionFilter.setFunction(expression.getFunction());
+            result.add(dimensionFilter);
+        }
+        return result;
+    }
+
+    private DateConf extractDateFilter(List<FieldExpression> fieldExpressions,
+            DataSetSchema dataSetSchema) {
+        List<FieldExpression> dateExpressions = fieldExpressions.stream().filter(
+                expression -> isPartitionDimension(dataSetSchema, expression.getFieldName()))
+                .collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(dateExpressions)) {
+            return null;
+        }
+        DateConf dateInfo = new DateConf();
+        dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
+        FieldExpression firstExpression = dateExpressions.get(0);
+
+        FilterOperatorEnum firstOperator =
+                FilterOperatorEnum.getSqlOperator(firstExpression.getOperator());
+        if (FilterOperatorEnum.EQUALS.equals(firstOperator)
+                && Objects.nonNull(firstExpression.getFieldValue())) {
+            dateInfo.setStartDate(firstExpression.getFieldValue().toString());
+            dateInfo.setEndDate(firstExpression.getFieldValue().toString());
+            dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
+            return dateInfo;
+        }
+        if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN,
+                FilterOperatorEnum.GREATER_THAN_EQUALS)) {
+            dateInfo.setStartDate(firstExpression.getFieldValue().toString());
+            if (hasSecondDate(dateExpressions)) {
+                dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString());
+            }
+        }
+        if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN,
+                FilterOperatorEnum.MINOR_THAN_EQUALS)) {
+            dateInfo.setEndDate(firstExpression.getFieldValue().toString());
+            if (hasSecondDate(dateExpressions)) {
+                dateInfo.setStartDate(dateExpressions.get(1).getFieldValue().toString());
+            }
+        }
+        return dateInfo;
+    }
+
+    private static boolean isPartitionDimension(DataSetSchema dataSetSchema, String sqlFieldName) {
+        if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) {
+            return true;
+        }
+        if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
+                || Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
+            return false;
+        }
+        return sqlFieldName.equalsIgnoreCase(dataSetSchema.getPartitionDimension().getName());
+    }
+
+    private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator,
+            FilterOperatorEnum... operatorEnums) {
+        return (Arrays.asList(operatorEnums).contains(firstOperator)
+                && Objects.nonNull(expression.getFieldValue()));
+    }
+
+    private boolean hasSecondDate(List<FieldExpression> dateExpressions) {
+        return dateExpressions.size() > 1
+                && Objects.nonNull(dateExpressions.get(1).getFieldValue());
+    }
+
+    private Map<String, SchemaElement> getNameToElement(DataSetSchema dsSchema) {
+        Set<SchemaElement> dimensions = dsSchema.getDimensions();
+        Set<SchemaElement> metrics = dsSchema.getMetrics();
+
+        List<SchemaElement> allElements = Lists.newArrayList();
+        allElements.addAll(dimensions);
+        allElements.addAll(metrics);
+        // support alias
+        return allElements.stream().flatMap(schemaElement -> {
+            Set<Pair<String, SchemaElement>> result = new HashSet<>();
+            result.add(Pair.of(schemaElement.getName(), schemaElement));
+            List<String> aliasList = schemaElement.getAlias();
+            if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) {
+                for (String alias : aliasList) {
+                    result.add(Pair.of(alias, schemaElement));
+                }
+            }
+            return result.stream();
+        }).collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (value1, value2) -> value2));
+    }
+
+    private void parseQueryType(SemanticParseInfo parseInfo) {
+        parseInfo.setQueryType(QueryType.DETAIL);
+        SqlInfo sqlInfo = parseInfo.getSqlInfo();
+        if (Objects.isNull(sqlInfo) || StringUtils.isBlank(sqlInfo.getCorrectedS2SQL())) {
+            parseInfo.setQueryType(QueryType.DETAIL);
+        }
+
+        // 2. AGG queryType
+        if (Objects.nonNull(sqlInfo) && StringUtils.isNotBlank(sqlInfo.getParsedS2SQL())
+                && SqlSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectedS2SQL())) {
+            parseInfo.setQueryType(QueryType.AGGREGATE);
+        }
+    }
+
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java

@@ -56,8 +56,7 @@ public class QueryRecommendProcessor implements ParseResultProcessor {
     private void updateChatQuery(ChatQueryDO chatQueryDO) {
         ChatQueryRepository chatQueryRepository = ContextUtils.getBean(ChatQueryRepository.class);
         UpdateWrapper<ChatQueryDO> updateWrapper = new UpdateWrapper<>();
-        updateWrapper.eq("question_id", chatQueryDO.getQuestionId());
+        updateWrapper.lambda().eq(ChatQueryDO::getQuestionId, chatQueryDO.getQuestionId());
-        updateWrapper.set("similar_queries", chatQueryDO.getSimilarQueries());
         chatQueryRepository.updateChatQuery(chatQueryDO, updateWrapper);
     }
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.server.agent.Agent;
 import com.tencent.supersonic.chat.server.agent.AgentToolType;
 import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.AuthType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -15,6 +16,7 @@ import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import java.util.List;
@@ -48,8 +50,11 @@ public class AgentController {
     }
 
     @RequestMapping("/getAgentList")
-    public List<Agent> getAgentList() {
+    public List<Agent> getAgentList(
-        return agentService.getAgents();
+            @RequestParam(value = "authType", required = false) AuthType authType,
+            HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
+        User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
+        return agentService.getAgents(user, authType);
     }
 
     @RequestMapping("/getToolTypes")

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java

@@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
 import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryCreateReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
 import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
-import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
+import com.tencent.supersonic.chat.server.pojo.ChatMemory;
 import com.tencent.supersonic.chat.server.service.MemoryService;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
@@ -32,7 +32,7 @@ public class MemoryController {
     public Boolean createMemory(@RequestBody ChatMemoryCreateReq chatMemoryCreateReq,
             HttpServletRequest request, HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        memoryService.createMemory(ChatMemoryDO.builder().agentId(chatMemoryCreateReq.getAgentId())
+        memoryService.createMemory(ChatMemory.builder().agentId(chatMemoryCreateReq.getAgentId())
                 .s2sql(chatMemoryCreateReq.getS2sql()).question(chatMemoryCreateReq.getQuestion())
                 .dbSchema(chatMemoryCreateReq.getDbSchema()).status(chatMemoryCreateReq.getStatus())
                 .humanReviewRet(MemoryReviewResult.POSITIVE).createdBy(user.getName())
@@ -49,7 +49,7 @@ public class MemoryController {
     }
 
     @RequestMapping("/pageMemories")
-    public PageInfo<ChatMemoryDO> pageMemories(@RequestBody PageMemoryReq pageMemoryReq) {
+    public PageInfo<ChatMemory> pageMemories(@RequestBody PageMemoryReq pageMemoryReq) {
         return memoryService.pageMemories(pageMemoryReq);
     }
 

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java

@@ -2,10 +2,12 @@ package com.tencent.supersonic.chat.server.service;
 
 import com.tencent.supersonic.chat.server.agent.Agent;
 import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.AuthType;
 
 import java.util.List;
 
 public interface AgentService {
+    List<Agent> getAgents(User user, AuthType authType);
 
     List<Agent> getAgents();
 

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/MemoryService.java

@@ -4,27 +4,22 @@ import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
 import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
 import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
-import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
+import com.tencent.supersonic.chat.server.pojo.ChatMemory;
 import com.tencent.supersonic.common.pojo.User;
 
 import java.util.List;
 
 public interface MemoryService {
-    void createMemory(ChatMemoryDO memory);
+    void createMemory(ChatMemory memory);
 
     void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user);
 
-    void updateMemory(ChatMemoryDO memory);
+    void updateMemory(ChatMemory memory);
-
-    void enableMemory(ChatMemoryDO memory);
-
-    void disableMemory(ChatMemoryDO memory);
 
     void batchDelete(List<Long> ids);
 
-    PageInfo<ChatMemoryDO> pageMemories(PageMemoryReq pageMemoryReq);
+    PageInfo<ChatMemory> pageMemories(PageMemoryReq pageMemoryReq);
 
-    List<ChatMemoryDO> getMemories(ChatMemoryFilter chatMemoryFilter);
+    List<ChatMemory> getMemories(ChatMemoryFilter chatMemoryFilter);
 
-    List<ChatMemoryDO> getMemoriesForLlmReview();
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java

@@ -6,14 +6,15 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
 import com.tencent.supersonic.chat.server.agent.Agent;
 import com.tencent.supersonic.chat.server.agent.VisualConfig;
 import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
 import com.tencent.supersonic.chat.server.persistence.mapper.AgentDOMapper;
+import com.tencent.supersonic.chat.server.pojo.ChatMemory;
 import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.chat.server.service.ChatQueryService;
 import com.tencent.supersonic.chat.server.service.MemoryService;
 import com.tencent.supersonic.common.config.ChatModel;
 import com.tencent.supersonic.common.pojo.ChatApp;
 import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.service.ChatModelService;
 import com.tencent.supersonic.common.util.JsonUtil;
 import lombok.extern.slf4j.Slf4j;
@@ -43,6 +44,27 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
 
     private ExecutorService executorService = Executors.newFixedThreadPool(1);
 
+    @Override
+    public List<Agent> getAgents(User user, AuthType authType) {
+        return getAgentDOList().stream().map(this::convert)
+                .filter(agent -> filterByAuth(agent, user, authType)).collect(Collectors.toList());
+    }
+
+    private boolean filterByAuth(Agent agent, User user, AuthType authType) {
+        if (user.isSuperAdmin() || user.getName().equals(agent.getCreatedBy())) {
+            return true;
+        }
+        authType = authType == null ? AuthType.VIEWER : authType;
+        switch (authType) {
+            case ADMIN:
+                return agent.contains(user, Agent::getAdmins);
+            case VIEWER:
+            default:
+                return agent.contains(user, Agent::getAdmins)
+                        || agent.contains(user, Agent::getViewers);
+        }
+    }
+
     @Override
     public List<Agent> getAgents() {
         return getAgentDOList().stream().map(this::convert).collect(Collectors.toList());
@@ -99,7 +121,7 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
         ChatMemoryFilter chatMemoryFilter =
                 ChatMemoryFilter.builder().agentId(agent.getId()).questions(examples).build();
         List<String> memoriesExisted = memoryService.getMemories(chatMemoryFilter).stream()
-                .map(ChatMemoryDO::getQuestion).collect(Collectors.toList());
+                .map(ChatMemory::getQuestion).collect(Collectors.toList());
         for (String example : examples) {
             if (memoriesExisted.contains(example)) {
                 continue;
@@ -135,6 +157,8 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
                 c.setChatModelConfig(chatModelService.getChatModel(c.getChatModelId()).getConfig());
             }
         });
+        agent.setAdmins(JsonUtil.toList(agentDO.getAdmin(), String.class));
+        agent.setViewers(JsonUtil.toList(agentDO.getViewer(), String.class));
         return agent;
     }
 
@@ -145,6 +169,8 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
         agentDO.setExamples(JsonUtil.toString(agent.getExamples()));
         agentDO.setChatModelConfig(JsonUtil.toString(agent.getChatAppConfig()));
         agentDO.setVisualConfig(JsonUtil.toString(agent.getVisualConfig()));
+        agentDO.setAdmin(JsonUtil.toString(agent.getAdmins()));
+        agentDO.setViewer(JsonUtil.toString(agent.getViewers()));
         if (agentDO.getStatus() == null) {
             agentDO.setStatus(1);
         }

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java

@@ -18,18 +18,12 @@ import com.tencent.supersonic.chat.server.service.ChatManageService;
 import com.tencent.supersonic.chat.server.service.ChatQueryService;
 import com.tencent.supersonic.chat.server.util.ComponentFactory;
 import com.tencent.supersonic.chat.server.util.QueryReqConverter;
-import com.tencent.supersonic.common.jsqlparser.FieldExpression;
+import com.tencent.supersonic.common.jsqlparser.*;
-import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
-import com.tencent.supersonic.headless.api.pojo.EntityInfo;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.SqlInfo;
@@ -50,11 +44,7 @@ import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.LongValue;
 import net.sf.jsqlparser.expression.StringValue;
-import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
+import net.sf.jsqlparser.expression.operators.relational.*;
-import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
-import net.sf.jsqlparser.expression.operators.relational.InExpression;
-import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
-import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
 import net.sf.jsqlparser.schema.Column;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -62,14 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -198,7 +181,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
             handleRuleQueryMode(semanticQuery, dataSetSchema, user);
         }
 
-        return executeQuery(semanticQuery, user, dataSetSchema);
+        return executeQuery(semanticQuery, user);
     }
 
     private List<String> getFieldsFromSql(SemanticParseInfo parseInfo) {
@@ -212,44 +195,44 @@ public class ChatQueryServiceImpl implements ChatQueryService {
     private void handleLLMQueryMode(ChatQueryDataReq chatQueryDataReq, SemanticQuery semanticQuery,
             DataSetSchema dataSetSchema, User user) throws Exception {
         SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
-        List<String> fields = getFieldsFromSql(parseInfo);
+        String rebuiltS2SQL;
-        if (checkMetricReplace(fields, chatQueryDataReq.getMetrics())) {
+        if (checkMetricReplace(chatQueryDataReq, parseInfo)) {
-            log.info("llm begin replace metrics!");
+            log.info("rebuild S2SQL with adjusted metrics!");
             SchemaElement metricToReplace = chatQueryDataReq.getMetrics().iterator().next();
-            replaceMetrics(parseInfo, metricToReplace);
+            rebuiltS2SQL = replaceMetrics(parseInfo, metricToReplace);
         } else {
-            log.info("llm begin revise filters!");
+            log.info("rebuild S2SQL with adjusted filters!");
-            String correctorSql = reviseCorrectS2SQL(chatQueryDataReq, parseInfo, dataSetSchema);
+            rebuiltS2SQL = replaceFilters(chatQueryDataReq, parseInfo, dataSetSchema);
-            parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql);
+        }
-            semanticQuery.setParseInfo(parseInfo);
+        // reset SqlInfo and request re-translation
+        parseInfo.getSqlInfo().setCorrectedS2SQL(rebuiltS2SQL);
+        parseInfo.getSqlInfo().setParsedS2SQL(rebuiltS2SQL);
+        parseInfo.getSqlInfo().setQuerySQL(null);
         SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
         SemanticTranslateResp explain = semanticLayerService.translate(semanticQueryReq, user);
         parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
     }
-    }
 
     private void handleRuleQueryMode(SemanticQuery semanticQuery, DataSetSchema dataSetSchema,
             User user) {
         log.info("rule begin replace metrics and revise filters!");
         validFilter(semanticQuery.getParseInfo().getDimensionFilters());
         validFilter(semanticQuery.getParseInfo().getMetricFilters());
-        semanticQuery.initS2Sql(dataSetSchema, user);
+        semanticQuery.buildS2Sql(dataSetSchema);
     }
 
-    private QueryResult executeQuery(SemanticQuery semanticQuery, User user,
+    private QueryResult executeQuery(SemanticQuery semanticQuery, User user) throws Exception {
-            DataSetSchema dataSetSchema) throws Exception {
         SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
         SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
         QueryResult queryResult = doExecution(semanticQueryReq, parseInfo.getQueryMode(), user);
         queryResult.setChatContext(semanticQuery.getParseInfo());
-        SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
-        EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
-        queryResult.setEntityInfo(entityInfo);
         parseInfo.getSqlInfo().setQuerySQL(queryResult.getQuerySql());
         return queryResult;
     }
 
-    private boolean checkMetricReplace(List<String> oriFields, Set<SchemaElement> metrics) {
+    private boolean checkMetricReplace(ChatQueryDataReq chatQueryDataReq, SemanticParseInfo parseInfo) {
+        List<String> oriFields = getFieldsFromSql(parseInfo);
+        Set<SchemaElement> metrics = chatQueryDataReq.getMetrics();
         if (CollectionUtils.isEmpty(oriFields) || CollectionUtils.isEmpty(metrics)) {
             return false;
         }
@@ -258,7 +241,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
         return !oriFields.containsAll(metricNames);
     }
 
-    private String reviseCorrectS2SQL(ChatQueryDataReq queryData, SemanticParseInfo parseInfo,
+    private String replaceFilters(ChatQueryDataReq queryData, SemanticParseInfo parseInfo,
                                   DataSetSchema dataSetSchema) {
         String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL();
         log.info("correctorSql before replacing:{}", correctorSql);
@@ -296,7 +279,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
         return correctorSql;
     }
 
-    private void replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) {
+    private String replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) {
         List<String> oriMetrics = parseInfo.getMetrics().stream().map(SchemaElement::getName)
                 .collect(Collectors.toList());
         String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL();
@@ -308,7 +291,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
             correctorSql = SqlReplaceHelper.replaceAggFields(correctorSql, fieldMap);
         }
         log.info("after replaceMetrics:{}", correctorSql);
-        parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql);
+        return correctorSql;
     }
 
     private QueryResult doExecution(SemanticQueryReq semanticQueryReq, String queryMode, User user)
@@ -483,6 +466,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
     }
 
     private void mergeParseInfo(SemanticParseInfo parseInfo, ChatQueryDataReq queryData) {
+        if (Objects.nonNull(queryData.getDateInfo())) {
+            parseInfo.setDateInfo(queryData.getDateInfo());
+        }
         if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
             return;
         }
@@ -498,9 +484,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
         if (!CollectionUtils.isEmpty(queryData.getMetricFilters())) {
             parseInfo.setMetricFilters(queryData.getMetricFilters());
         }
-        if (Objects.nonNull(queryData.getDateInfo())) {
+
-            parseInfo.setDateInfo(queryData.getDateInfo());
-        }
         parseInfo.setSqlInfo(new SqlInfo());
     }
 

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java

@@ -7,7 +7,6 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
 import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
-import com.tencent.supersonic.chat.api.pojo.request.Entity;
 import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
 import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
@@ -16,7 +15,6 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
 import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
 import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
 import com.tencent.supersonic.chat.api.pojo.response.ChatDetailRichConfigResp;
-import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp;
 import com.tencent.supersonic.chat.api.pojo.response.ItemVisibilityInfo;
 import com.tencent.supersonic.chat.server.config.ChatConfig;
 import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository;
@@ -88,16 +86,6 @@ public class ConfigServiceImpl implements ConfigService {
         return configEditCmd.getId();
     }
 
-    public ItemNameVisibilityInfo getVisibilityByModelId(Long modelId) {
-        ChatConfigResp chatConfigResp = fetchConfigByModelId(modelId);
-        ChatConfig chatConfig = new ChatConfig();
-        chatConfig.setModelId(modelId);
-        chatConfig.setChatAggConfig(chatConfigResp.getChatAggConfig());
-        chatConfig.setChatDetailConfig(chatConfigResp.getChatDetailConfig());
-        ItemNameVisibilityInfo itemNameVisibility = getItemNameVisibility(chatConfig);
-        return itemNameVisibility;
-    }
-
     public ItemNameVisibilityInfo getItemNameVisibility(ChatConfig chatConfig) {
         Long modelId = chatConfig.getModelId();
 
@@ -240,19 +228,6 @@ public class ConfigServiceImpl implements ConfigService {
         return detailRichConfig;
     }
 
-    private EntityRichInfoResp generateRichEntity(Entity entity, DataSetSchema modelSchema) {
-        EntityRichInfoResp entityRichInfo = new EntityRichInfoResp();
-        if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) {
-            return entityRichInfo;
-        }
-        BeanUtils.copyProperties(entity, entityRichInfo);
-        Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
-                Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
-
-        entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId()));
-        return entityRichInfo;
-    }
-
     private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema,
             ChatConfigResp chatConfigResp) {
         if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {
@@ -327,7 +302,7 @@ public class ConfigServiceImpl implements ConfigService {
         }
         Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
                 Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
-        knowledgeInfos.stream().forEach(knowledgeInfo -> {
+        knowledgeInfos.forEach(knowledgeInfo -> {
             if (Objects.nonNull(knowledgeInfo)) {
                 SchemaElement dimSchemaResp = dimIdAndRespPair.get(knowledgeInfo.getItemId());
                 if (Objects.nonNull(dimSchemaResp)) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java

@@ -3,12 +3,14 @@ package com.tencent.supersonic.chat.server.service.impl;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
+import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
 import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
 import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
 import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
 import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
 import com.tencent.supersonic.chat.server.persistence.repository.ChatMemoryRepository;
+import com.tencent.supersonic.chat.server.pojo.ChatMemory;
 import com.tencent.supersonic.chat.server.service.MemoryService;
 import com.tencent.supersonic.common.config.EmbeddingConfig;
 import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
@@ -16,12 +18,15 @@ import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.service.ExemplarService;
 import com.tencent.supersonic.common.util.BeanMapper;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
 import java.util.Date;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 @Service
 public class MemoryServiceImpl implements MemoryService {
@@ -36,34 +41,36 @@ public class MemoryServiceImpl implements MemoryService {
     private EmbeddingConfig embeddingConfig;
 
     @Override
-    public void createMemory(ChatMemoryDO memory) {
+    public void createMemory(ChatMemory memory) {
         // if an existing enabled memory has the same question, just skip
-        List<ChatMemoryDO> memories =
+        List<ChatMemory> memories =
                 getMemories(ChatMemoryFilter.builder().agentId(memory.getAgentId())
                         .question(memory.getQuestion()).status(MemoryStatus.ENABLED).build());
-        if (memories.size() == 0) {
+        if (memories.isEmpty()) {
-            chatMemoryRepository.createMemory(memory);
+            ChatMemoryDO memoryDO = getMemoryDO(memory);
+            chatMemoryRepository.createMemory(memoryDO);
         }
     }
 
     @Override
     public void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user) {
         ChatMemoryDO chatMemoryDO = chatMemoryRepository.getMemory(chatMemoryUpdateReq.getId());
+        boolean hadEnabled =
+                MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim());
         chatMemoryDO.setUpdatedBy(user.getName());
         chatMemoryDO.setUpdatedAt(new Date());
         BeanMapper.mapper(chatMemoryUpdateReq, chatMemoryDO);
-        boolean hadEnabled = MemoryStatus.ENABLED.equals(chatMemoryDO.getStatus());
         if (MemoryStatus.ENABLED.equals(chatMemoryUpdateReq.getStatus()) && !hadEnabled) {
             enableMemory(chatMemoryDO);
         } else if (MemoryStatus.DISABLED.equals(chatMemoryUpdateReq.getStatus()) && hadEnabled) {
             disableMemory(chatMemoryDO);
         }
-        updateMemory(chatMemoryDO);
+        chatMemoryRepository.updateMemory(chatMemoryDO);
     }
 
     @Override
-    public void updateMemory(ChatMemoryDO memory) {
+    public void updateMemory(ChatMemory memory) {
-        chatMemoryRepository.updateMemory(memory);
+        chatMemoryRepository.updateMemory(getMemoryDO(memory));
     }
 
     @Override
@@ -72,7 +79,7 @@ public class MemoryServiceImpl implements MemoryService {
     }
 
     @Override
-    public PageInfo<ChatMemoryDO> pageMemories(PageMemoryReq pageMemoryReq) {
+    public PageInfo<ChatMemory> pageMemories(PageMemoryReq pageMemoryReq) {
         ChatMemoryFilter chatMemoryFilter = pageMemoryReq.getChatMemoryFilter();
         chatMemoryFilter.setSort(pageMemoryReq.getSort());
         chatMemoryFilter.setOrderCondition(pageMemoryReq.getOrderCondition());
@@ -81,7 +88,7 @@ public class MemoryServiceImpl implements MemoryService {
     }
 
     @Override
-    public List<ChatMemoryDO> getMemories(ChatMemoryFilter chatMemoryFilter) {
+    public List<ChatMemory> getMemories(ChatMemoryFilter chatMemoryFilter) {
         QueryWrapper<ChatMemoryDO> queryWrapper = new QueryWrapper<>();
         if (chatMemoryFilter.getAgentId() != null) {
             queryWrapper.lambda().eq(ChatMemoryDO::getAgentId, chatMemoryFilter.getAgentId());
@@ -109,32 +116,52 @@ public class MemoryServiceImpl implements MemoryService {
             queryWrapper.orderBy(true, chatMemoryFilter.isAsc(),
                     chatMemoryFilter.getOrderCondition());
         }
-        return chatMemoryRepository.getMemories(queryWrapper);
+        List<ChatMemoryDO> chatMemoryDOS = chatMemoryRepository.getMemories(queryWrapper);
+        return chatMemoryDOS.stream().map(this::getMemory).collect(Collectors.toList());
     }
 
-    @Override
+    private void enableMemory(ChatMemoryDO memory) {
-    public List<ChatMemoryDO> getMemoriesForLlmReview() {
+        memory.setStatus(MemoryStatus.ENABLED.toString());
-        QueryWrapper<ChatMemoryDO> queryWrapper = new QueryWrapper<>();
-        queryWrapper.lambda().eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING)
-                .isNull(ChatMemoryDO::getLlmReviewRet);
-        return chatMemoryRepository.getMemories(queryWrapper);
-    }
-
-    @Override
-    public void enableMemory(ChatMemoryDO memory) {
-        memory.setStatus(MemoryStatus.ENABLED);
         exemplarService.storeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
                 Text2SQLExemplar.builder().question(memory.getQuestion())
                         .sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
                         .sql(memory.getS2sql()).build());
     }
 
-    @Override
+    private void disableMemory(ChatMemoryDO memory) {
-    public void disableMemory(ChatMemoryDO memory) {
+        memory.setStatus(MemoryStatus.DISABLED.toString());
-        memory.setStatus(MemoryStatus.DISABLED);
         exemplarService.removeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
                 Text2SQLExemplar.builder().question(memory.getQuestion())
                         .sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
                         .sql(memory.getS2sql()).build());
     }
+
+    private ChatMemoryDO getMemoryDO(ChatMemory memory) {
+        ChatMemoryDO memoryDO = new ChatMemoryDO();
+        BeanUtils.copyProperties(memory, memoryDO);
+        memoryDO.setStatus(memory.getStatus().toString().trim());
+        if (Objects.nonNull(memory.getHumanReviewRet())) {
+            memoryDO.setHumanReviewRet(memory.getHumanReviewRet().toString().trim());
+        }
+        if (Objects.nonNull(memory.getLlmReviewRet())) {
+            memoryDO.setLlmReviewRet(memory.getLlmReviewRet().toString().trim());
+        }
+
+        return memoryDO;
+    }
+
+    private ChatMemory getMemory(ChatMemoryDO memoryDO) {
+        ChatMemory memory = new ChatMemory();
+        BeanUtils.copyProperties(memoryDO, memory);
+        memory.setStatus(MemoryStatus.valueOf(memoryDO.getStatus().trim()));
+        if (Objects.nonNull(memoryDO.getHumanReviewRet())) {
+            memory.setHumanReviewRet(
+                    MemoryReviewResult.valueOf(memoryDO.getHumanReviewRet().trim()));
+        }
+        if (Objects.nonNull(memoryDO.getLlmReviewRet())) {
+            memory.setLlmReviewRet(MemoryReviewResult.valueOf(memoryDO.getLlmReviewRet().trim()));
+        }
+        return memory;
+    }
+
 }

chat/server/src/main/resources/mapper/ChatConfigMapper.xml

@@ -20,54 +20,6 @@
         <result column="updated_at" property="updatedAt"/>
     </resultMap>
 
-    <insert id="addConfig"
-            parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO"
-            useGeneratedKeys="true" keyProperty="id">
-    insert into s2_chat_config
-    (
-        model_id, `chat_detail_config`, chat_agg_config,  recommended_questions, status, llm_examples, created_by, updated_by, created_at, updated_at
-    )
-    values
-    (
-        #{modelId}, #{chatDetailConfig}, #{chatAggConfig}, #{recommendedQuestions}, #{status}, #{llmExamples}, #{createdBy}, #{updatedBy}, #{createdAt}, #{updatedAt}
-    )
-    </insert>
-
-
-    <update id="editConfig">
-        update s2_chat_config
-        <set>
-            `updated_at` = #{updatedAt} ,
-            <if test="chatDetailConfig != null and chatDetailConfig != ''">
-                `chat_detail_config` = #{chatDetailConfig} ,
-            </if>
-            <if test="chatAggConfig != null and chatAggConfig != ''">
-                chat_agg_config = #{chatAggConfig} ,
-            </if>
-            <if test="recommendedQuestions != null and recommendedQuestions != ''">
-                recommended_questions = #{recommendedQuestions} ,
-            </if>
-            <if test="status != null and status != ''">
-                status = #{status} ,
-            </if>
-            <if test="updatedBy != null and updatedBy != ''">
-                updated_by = #{updatedBy} ,
-            </if>
-            <if test="llmExamples != null and llmExamples != ''">
-                llm_examples = #{llmExamples} ,
-            </if>
-        </set>
-
-        <where>
-            <if test="id != null and id != ''">
-                id = #{id}
-            </if>
-            <if test="modelId != null and modelId != ''">
-                and model_id = #{modelId}
-            </if>
-        </where>
-    </update>
-
     <select id="search" resultMap="chaConfigDO">
         select *
         from s2_chat_config

chat/server/src/main/resources/mapper/ChatContextMapper.xml

@@ -20,11 +20,4 @@
         from s2_chat_context where chat_id=#{chatId} limit 1
     </select>
 
-    <insert id="addContext" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO" >
-        insert into s2_chat_context  (chat_id,user,query_text,semantic_parse) values (#{chatId}, #{user},#{queryText}, #{semanticParse})
-    </insert>
-    <update id="updateContext">
-        update  s2_chat_context  set query_text=#{queryText},semantic_parse=#{semanticParse} where chat_id=#{chatId}
-    </update>
-
 </mapper>

common/src/main/java/com/hankcs/hanlp/seg/common/Term.java

@@ -19,19 +19,6 @@ public class Term {
         this.nature = nature;
     }
 
-    public Term(String word, Nature nature, int offset) {
-        this.word = word;
-        this.nature = nature;
-        this.offset = offset;
-    }
-
-    public Term(String word, Nature nature, int offset, int frequency) {
-        this.word = word;
-        this.nature = nature;
-        this.offset = offset;
-        this.frequency = frequency;
-    }
-
     public int length() {
         return this.word.length();
     }

common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java

@@ -38,14 +38,14 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
             new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", MODULE_NAME);
 
     public static final Parameter EMBEDDING_STORE_DIMENSION =
-            new Parameter("s2.embedding.store.dimension", "", "纬度", "", "number", MODULE_NAME, null,
+            new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME,
-                    getDimensionDependency());
+                    null, getDimensionDependency());
     public static final Parameter EMBEDDING_STORE_DATABASE_NAME =
             new Parameter("s2.embedding.store.databaseName", "", "DatabaseName", "", "string",
                     MODULE_NAME, null, getDatabaseNameDependency());
 
-    public static final Parameter EMBEDDING_STORE_POST = new Parameter("s2.embedding.store.post",
+    public static final Parameter EMBEDDING_STORE_POST = new Parameter("s2.embedding.store.port",
-            "", "端口", "", "number", MODULE_NAME, null, getPostDependency());
+            "", "端口", "", "number", MODULE_NAME, null, getPortDependency());
 
     public static final Parameter EMBEDDING_STORE_USER = new Parameter("s2.embedding.store.user",
             "", "用户名", "", "string", MODULE_NAME, null, getUserDependency());
@@ -101,10 +101,8 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
 
     private static List<Parameter.Dependency> getApiKeyDependency() {
         return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
-                Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
+                Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
-                        EmbeddingStoreType.PGVECTOR.name()),
+                ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO));
-                ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO,
-                        EmbeddingStoreType.PGVECTOR.name(), DEMO));
     }
 
     private static List<Parameter.Dependency> getPathDependency() {
@@ -118,7 +116,7 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
                 Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
                         EmbeddingStoreType.PGVECTOR.name()),
                 ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "384",
-                        EmbeddingStoreType.PGVECTOR.name(), "768"));
+                        EmbeddingStoreType.PGVECTOR.name(), "512"));
     }
 
     private static List<Parameter.Dependency> getDatabaseNameDependency() {
@@ -129,7 +127,7 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
                         EmbeddingStoreType.PGVECTOR.name(), "postgres"));
     }
 
-    private static List<Parameter.Dependency> getPostDependency() {
+    private static List<Parameter.Dependency> getPortDependency() {
         return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
                 Lists.newArrayList(EmbeddingStoreType.PGVECTOR.name()),
                 ImmutableMap.of(EmbeddingStoreType.PGVECTOR.name(), "54333"));
@@ -140,12 +138,14 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
                 Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
                         EmbeddingStoreType.PGVECTOR.name()),
                 ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus",
-                        EmbeddingStoreType.PGVECTOR.name(), "pgvector"));
+                        EmbeddingStoreType.PGVECTOR.name(), "postgres"));
     }
 
     private static List<Parameter.Dependency> getPasswordDependency() {
         return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
-                Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
+                Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
-                ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus"));
+                        EmbeddingStoreType.PGVECTOR.name()),
+                ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus",
+                        EmbeddingStoreType.PGVECTOR.name(), "postgres"));
     }
 }

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java

@@ -528,7 +528,7 @@ public class SqlReplaceHelper {
         }
     }
 
-    private static Select replaceAggAliasOrderItem(Select selectStatement) {
+    private static Select replaceAggAliasOrderbyField(Select selectStatement) {
         if (selectStatement instanceof PlainSelect) {
             PlainSelect plainSelect = (PlainSelect) selectStatement;
             if (Objects.nonNull(plainSelect.getOrderByElements())) {
@@ -564,15 +564,15 @@ public class SqlReplaceHelper {
             if (plainSelect.getFromItem() instanceof ParenthesedSelect) {
                 ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem();
                 parenthesedSelect
-                        .setSelect(replaceAggAliasOrderItem(parenthesedSelect.getSelect()));
+                        .setSelect(replaceAggAliasOrderbyField(parenthesedSelect.getSelect()));
             }
             return selectStatement;
         }
         return selectStatement;
     }
 
-    public static String replaceAggAliasOrderItem(String sql) {
+    public static String replaceAggAliasOrderbyField(String sql) {
-        Select selectStatement = replaceAggAliasOrderItem(SqlSelectHelper.getSelect(sql));
+        Select selectStatement = replaceAggAliasOrderbyField(SqlSelectHelper.getSelect(sql));
         return selectStatement.toString();
     }
 

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlValidHelper.java

@@ -74,7 +74,6 @@ public class SqlValidHelper {
             CCJSqlParserUtil.parse(sql);
             return true;
         } catch (Exception e) {
-            log.error("isValidSQL parse:{}", e);
             return false;
         }
     }

common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java

@@ -8,7 +8,6 @@ import lombok.Data;
 @Builder
 public class DataItem {
 
-    /** * This field uses an underscore (_) at the end. */
     private String id;
 
     private String bizName;
@@ -19,9 +18,10 @@ public class DataItem {
 
     private TypeEnums type;
 
-    /** * This field uses an underscore (_) at the end. */
     private String modelId;
 
+    private String domainId;
+
     private String defaultAgg;
 
     public String getNewName() {

common/src/main/java/com/tencent/supersonic/common/pojo/Filter.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.common.pojo;
 
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
@@ -10,6 +11,7 @@ import java.util.List;
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
+@Builder
 public class Filter {
 
     private Relation relation = Relation.FILTER;

common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.common.pojo;
 
+import com.google.common.collect.Lists;
 import lombok.Data;
 
 import java.util.List;
@@ -18,5 +19,5 @@ public class ModelRela extends RecordInfo {
     // left join, inner join, right join, outer join
     private String joinType;
 
-    private List<JoinCondition> joinConditions;
+    private List<JoinCondition> joinConditions = Lists.newArrayList();
 }

common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java

@@ -1,5 +1,5 @@
 package com.tencent.supersonic.common.pojo.enums;
 
 public enum AuthType {
-    VISIBLE, ADMIN
+    VIEWER, ADMIN
 }

common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java

@@ -14,8 +14,6 @@ public enum DictWordType {
 
     DATASET("dataSet"),
 
-    ENTITY("entity"),
-
     NUMBER("m"),
 
     TAG("tag"),

common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java

@@ -8,7 +8,8 @@ public enum EngineType {
     KAFKA(4, "kafka"),
     H2(5, "h2"),
     POSTGRESQL(6, "postgresql"),
-    OTHER(7, "other");
+    OTHER(7, "other"),
+    DUCKDB(8, "duckdb");
 
     private Integer code;
 

common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java

@@ -5,9 +5,7 @@ public enum QueryType {
     /** queries with aggregation (optionally slice and dice by dimensions) */
     AGGREGATE,
     /** queries with field selection */
-    DETAIL,
+    DETAIL;
-    /** queries with ID-based entity selection */
-    ID;
 
     public boolean isNativeAggQuery() {
         return DETAIL.equals(this);

common/src/main/java/com/tencent/supersonic/common/pojo/enums/Text2SQLType.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.common.pojo.enums;
 
 public enum Text2SQLType {
-    ONLY_RULE, LLM_OR_RULE;
+    ONLY_RULE, LLM_OR_RULE, NONE;
 
     public boolean enableLLM() {
         return this.equals(LLM_OR_RULE);

common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java

@@ -1,5 +1,5 @@
 package com.tencent.supersonic.common.pojo.enums;
 
 public enum TypeEnums {
-    METRIC, DIMENSION, TAG_OBJECT, TAG, DOMAIN, ENTITY, DATASET, MODEL, UNKNOWN
+    METRIC, DIMENSION, TAG, DOMAIN, DATASET, MODEL, UNKNOWN
 }

common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java

@@ -4,6 +4,7 @@ import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
 import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -32,14 +33,9 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
 @Data
 public class DateModeUtils {
 
-    @Value("${s2.query.parameter.sys.date:sys_imp_date}")
+    private final String sysDateCol = TimeDimensionEnum.DAY.getName();
-    private String sysDateCol;
+    private final String sysDateMonthCol = TimeDimensionEnum.MONTH.getName();
-
+    private final String sysDateWeekCol = TimeDimensionEnum.WEEK.getName();
-    @Value("${s2.query.parameter.sys.month:sys_imp_month}")
-    private String sysDateMonthCol;
-
-    @Value("${s2.query.parameter.sys.month:sys_imp_week}")
-    private String sysDateWeekCol;
 
     @Value("${s2.query.parameter.sys.zipper.begin:start_}")
     private String sysZipperDateColBegin;

common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java

@@ -16,6 +16,7 @@ import java.time.temporal.TemporalAdjuster;
 import java.time.temporal.TemporalAdjusters;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
 import java.util.Objects;
@@ -201,6 +202,13 @@ public class DateUtils {
         return false;
     }
 
+    public static Long calculateDiffMs(Date createAt) {
+        Calendar calendar = Calendar.getInstance();
+        Date now = calendar.getTime();
+        long milliseconds = now.getTime() - createAt.getTime();
+        return milliseconds;
+    }
+
     public static boolean isDateString(String value, String format) {
         try {
             DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format);

common/src/main/java/dev/langchain4j/pgvector/spring/PgvectorEmbeddingStoreFactory.java

@@ -26,7 +26,7 @@ public class PgvectorEmbeddingStoreFactory extends BaseEmbeddingStoreFactory {
         embeddingStore.setPort(storeConfig.getPost());
         embeddingStore.setDatabase(storeConfig.getDatabaseName());
         embeddingStore.setUser(storeConfig.getUser());
-        embeddingStore.setPassword(storeConfig.getApiKey());
+        embeddingStore.setPassword(storeConfig.getPassword());
         return embeddingStore;
     }
 

common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java

@@ -1,6 +1,7 @@
 package dev.langchain4j.store.embedding;
 
 import com.alibaba.fastjson.JSONObject;
+import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DataItem;
 import dev.langchain4j.data.document.Metadata;
 import dev.langchain4j.data.segment.TextSegment;
@@ -17,10 +18,18 @@ public class TextSegmentConvert {
     public static final String QUERY_ID = "queryId";
 
     public static List<TextSegment> convertToEmbedding(List<DataItem> dataItems) {
-        return dataItems.stream().map(dataItem -> {
+        return dataItems.stream().map(item -> {
-            Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class);
+            // suffix with underscore to avoid embedding issue
-            TextSegment textSegment = TextSegment.from(dataItem.getName(), new Metadata(meta));
+            DataItem newItem = DataItem.builder().domainId(item.getDomainId())
-            addQueryId(textSegment, dataItem.getId() + dataItem.getType().name().toLowerCase());
+                    .bizName(item.getBizName()).type(item.getType()).newName(item.getNewName())
+                    .defaultAgg(item.getDefaultAgg()).name(item.getName())
+                    .id(item.getId() + Constants.UNDERLINE)
+                    .modelId(item.getModelId() + Constants.UNDERLINE)
+                    .domainId(item.getDomainId() + Constants.UNDERLINE).build();
+
+            Map meta = JSONObject.parseObject(JSONObject.toJSONString(newItem), Map.class);
+            TextSegment textSegment = TextSegment.from(newItem.getName(), new Metadata(meta));
+            addQueryId(textSegment, newItem.getId() + newItem.getType().name().toLowerCase());
             return textSegment;
         }).collect(Collectors.toList());
     }

common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java

@@ -325,10 +325,10 @@ class SqlReplaceHelperTest {
     }
 
     @Test
-    void testReplaceAggAliasOrderItem() {
+    void testReplaceAggAliasOrderbyField() {
         String sql = "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数  "
                 + "GROUP BY 部门 ORDER BY SUM(访问次数) DESC LIMIT 10) AS top10";
-        String replaceSql = SqlReplaceHelper.replaceAggAliasOrderItem(sql);
+        String replaceSql = SqlReplaceHelper.replaceAggAliasOrderbyField(sql);
         Assert.assertEquals(
                 "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 "
                         + "GROUP BY 部门 ORDER BY 2 DESC LIMIT 10) AS top10",

docker/Dockerfile

@@ -7,14 +7,13 @@ WORKDIR /usr/src/app
 # Argument to pass in the supersonic version at build time
 ARG SUPERSONIC_VERSION
 
-# Install necessary packages, including MySQL client
+RUN apt-get update
-RUN apt-get update && \
+
-    apt-get install -y default-mysql-client unzip && \
+# Install necessary packages, including Postgres client
-    rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y postgresql-client
 
 # Install the vim editor.
-RUN apt-get update && \
+RUN apt-get update && apt-get install -y vim && \
-    apt-get install -y vim && \
     rm -rf /var/lib/apt/lists/*
 
 # Update the package list and install iputils-ping.
@@ -40,4 +39,4 @@ WORKDIR /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION}
 EXPOSE 9080
 # Command to run the supersonic daemon
 RUN chmod +x bin/supersonic-daemon.sh
-CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone prd && tail -f /dev/null"]
+CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone docker && tail -f /dev/null"]

docker/DockerfileS2

@@ -0,0 +1,32 @@
+# Use an official OpenJDK runtime as a parent image
+FROM supersonicbi/supersonic:0.9.8
+
+# Set the working directory in the container
+WORKDIR /usr/src/app
+
+# Argument to pass in the supersonic version at build time
+ARG SUPERSONIC_VERSION
+
+# Install necessary packages, including Postgres client
+RUN apt-get install -y postgresql-client
+
+RUN rm /usr/src/app/supersonic-standalone-latest
+
+# Copy the supersonic standalone zip file into the container
+COPY assembly/build/supersonic-standalone-${SUPERSONIC_VERSION}.zip .
+
+# Unzip the supersonic standalone zip
+RUN unzip supersonic-standalone-${SUPERSONIC_VERSION}.zip && \
+    rm supersonic-standalone-${SUPERSONIC_VERSION}.zip
+
+# Create a symbolic link to the supersonic installation directory
+RUN ln -s /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION} /usr/src/app/supersonic-standalone-latest
+
+# Set the working directory to the supersonic installation directory
+WORKDIR /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION}
+
+# Expose the default port
+EXPOSE 9080
+# Command to run the supersonic daemon
+RUN chmod +x bin/supersonic-daemon.sh
+CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone docker && tail -f /dev/null"]

docker/docker-compose-run.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+SUPERSONIC_VERSION=0.9.10-SNAPSHOT docker-compose -f docker-compose.yml -p supersonic up

docker/docker-compose.yml

@@ -1,51 +1,28 @@
 services:
-  chroma:
+  postgres:
-    image: chromadb/chroma:0.5.3
+    image: pgvector/pgvector:pg17
     privileged: true
-    container_name: supersonic_chroma
+    container_name: supersonic_postgres
-    ports:
-      - "8000:8000"
-    volumes:
-      - chroma_data:/chroma
-    networks:
-      - supersonic_network
-    dns:
-      - 114.114.114.114
-      - 8.8.8.8
-      - 8.8.4.4
-    healthcheck:
-      test: ["CMD", "curl", "http://0.0.0.0:8000"]
-      interval: 10s
-      timeout: 5s
-      retries: 10
-
-  mysql:
-    image: mysql:8.0
-    privileged: true
-    container_name: supersonic_mysql
     environment:
       LANG: 'C.UTF-8'  # 设置环境变量
-      MYSQL_ROOT_PASSWORD: root_password
+      POSTGRES_ROOT_PASSWORD: root_password
-      MYSQL_DATABASE: supersonic_db
+      POSTGRES_DATABASE: postgres
-      MYSQL_USER: supersonic_user
+      POSTGRES_USER: supersonic_user
-      MYSQL_PASSWORD: supersonic_password
+      POSTGRES_PASSWORD: supersonic_password
     ports:
-      - "13306:3306"
+      - "15432:5432"
     volumes:
-      - mysql_data:/var/lib/mysql
+      - postgres_data:/var/lib/postgresql
     networks:
       - supersonic_network
     dns:
       - 114.114.114.114
       - 8.8.8.8
       - 8.8.4.4
-    depends_on:
-      chroma:
-        condition: service_healthy
     healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
+      test: ["CMD-SHELL", "sh -c 'pg_isready -U supersonic_user -d postgres'"]
-      interval: 10s
+      interval: 30s
-      timeout: 5s
+      timeout: 10s
       retries: 5
 
   db_init:
@@ -53,20 +30,21 @@ services:
     privileged: true
     container_name: supersonic_db_init
     depends_on:
-      mysql:
+      postgres:
         condition: service_healthy
     networks:
       - supersonic_network
     command: >
       sh -c "
-      sleep 15 &&
+        if ! PGPASSWORD=supersonic_password psql -h supersonic_postgres -U supersonic_user -d postgres -c 'select * from s2_database limit 1' > /dev/null;
-      if ! mysql -h supersonic_mysql -usupersonic_user -psupersonic_password -e 'use supersonic_db; show tables;' | grep -q 's2_database'; then
+        then
-        mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql.sql &&
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-postgres.sql
-        mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql.sql
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-postgres-demo.sql
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-postgres.sql
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-postgres-demo.sql 
         else
           echo 'Database already initialized.'
-      fi
+        fi"
-      "
     dns:
       - 114.114.114.114
       - 8.8.8.8
@@ -77,17 +55,14 @@ services:
     privileged: true
     container_name: supersonic_standalone
     environment:
-      DB_HOST: supersonic_mysql
+      DB_HOST: supersonic_postgres
-      DB_NAME: supersonic_db
+      DB_NAME: postgres
       DB_USERNAME: supersonic_user
       DB_PASSWORD: supersonic_password
-      CHROMA_HOST: supersonic_chroma
     ports:
       - "9080:9080"
     depends_on:
-      chroma:
+      postgres:
-        condition: service_healthy
-      mysql:
         condition: service_healthy
       db_init:
         condition: service_completed_successfully
@@ -110,8 +85,7 @@ services:
       #          propagation: rprivate
       #          create_host_path: true
 volumes:
-  mysql_data:
+  postgres_data:
-  chroma_data:
   supersonic_data:
 
 networks:

docker/docker-publish.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Exit immediately if a command exits with a non-zero status
+set -e
+VERSION=$1
+
+# Image name
+IMAGE_NAME="supersonicbi/supersonic"
+
+# Default tag is latest
+TAGS="latest"
+
+# If VERSION is provided, add it to TAGS and tag the image as latest
+if [ -n "$VERSION" ]; then
+  TAGS="$TAGS $VERSION"
+  docker tag $IMAGE_NAME:$VERSION $IMAGE_NAME:latest
+fi
+
+# Push Docker images
+for TAG in $TAGS; do
+  echo "Pushing Docker image $IMAGE_NAME:$TAG"
+  docker push $IMAGE_NAME:$TAG
+done
+
+echo "Docker images pushed successfully."

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java

@@ -15,7 +15,7 @@ public class ColumnSchema {
 
     private FieldType filedType;
 
-    private AggOperatorEnum agg;
+    private AggOperatorEnum agg = AggOperatorEnum.SUM;
 
     private String name;
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo;
 
+import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -14,4 +15,6 @@ public class DBColumn {
     private String dataType;
 
     private String comment;
+
+    private FieldType fieldType;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo;
 
 import lombok.Data;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.Serializable;
@@ -9,7 +8,6 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -24,16 +22,16 @@ public class DataSetSchema implements Serializable {
     private Set<SchemaElement> tags = new HashSet<>();
     private Set<SchemaElement> dimensionValues = new HashSet<>();
     private Set<SchemaElement> terms = new HashSet<>();
-    private SchemaElement entity = new SchemaElement();
     private QueryConfig queryConfig;
 
+    public Long getDataSetId() {
+        return dataSet.getDataSetId();
+    }
+
     public SchemaElement getElement(SchemaElementType elementType, long elementID) {
         Optional<SchemaElement> element = Optional.empty();
 
         switch (elementType) {
-            case ENTITY:
-                element = Optional.ofNullable(entity);
-                break;
             case DATASET:
                 element = Optional.of(dataSet);
                 break;
@@ -55,11 +53,7 @@ public class DataSetSchema implements Serializable {
             default:
         }
 
-        if (element.isPresent()) {
+        return element.orElse(null);
-            return element.get();
-        } else {
-            return null;
-        }
     }
 
     public Map<String, String> getBizNameToName() {
@@ -70,7 +64,7 @@ public class DataSetSchema implements Serializable {
                 SchemaElement::getName, (k1, k2) -> k1));
     }
 
-    public TimeDefaultConfig getTagTypeTimeDefaultConfig() {
+    public TimeDefaultConfig getDetailTypeTimeDefaultConfig() {
         if (queryConfig == null) {
             return null;
         }
@@ -90,45 +84,6 @@ public class DataSetSchema implements Serializable {
         return queryConfig.getAggregateTypeDefaultConfig().getTimeDefaultConfig();
     }
 
-    public DetailTypeDefaultConfig getTagTypeDefaultConfig() {
-        if (queryConfig == null) {
-            return null;
-        }
-        return queryConfig.getDetailTypeDefaultConfig();
-    }
-
-    public List<SchemaElement> getTagDefaultDimensions() {
-        DetailTypeDefaultConfig detailTypeDefaultConfig = getTagTypeDefaultConfig();
-        if (Objects.isNull(detailTypeDefaultConfig)
-                || Objects.isNull(detailTypeDefaultConfig.getDefaultDisplayInfo())) {
-            return new ArrayList<>();
-        }
-        if (CollectionUtils
-                .isNotEmpty(detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) {
-            return detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds().stream()
-                    .map(id -> {
-                        SchemaElement metric = getElement(SchemaElementType.METRIC, id);
-                        return metric;
-                    }).filter(Objects::nonNull).collect(Collectors.toList());
-        }
-        return new ArrayList<>();
-    }
-
-    public List<SchemaElement> getTagDefaultMetrics() {
-        DetailTypeDefaultConfig detailTypeDefaultConfig = getTagTypeDefaultConfig();
-        if (Objects.isNull(detailTypeDefaultConfig)
-                || Objects.isNull(detailTypeDefaultConfig.getDefaultDisplayInfo())) {
-            return new ArrayList<>();
-        }
-        if (CollectionUtils
-                .isNotEmpty(detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) {
-            return detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
-                    .map(id -> getElement(SchemaElementType.DIMENSION, id)).filter(Objects::nonNull)
-                    .collect(Collectors.toList());
-        }
-        return new ArrayList<>();
-    }
-
     public boolean containsPartitionDimensions() {
         return dimensions.stream().anyMatch(SchemaElement::isPartitionTime);
     }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DbSchema.java

@@ -17,5 +17,7 @@ public class DbSchema {
 
     private String sql;
 
+    private String ddl;
+
     private List<DBColumn> dbColumns;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java

@@ -1,16 +0,0 @@
-package com.tencent.supersonic.headless.api.pojo;
-
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-public class DefaultDisplayInfo implements Serializable {
-
-    // When displaying tag selection results, the information displayed by default
-    private List<Long> dimensionIds = new ArrayList<>();
-
-    private List<Long> metricIds = new ArrayList<>();
-}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DetailTypeDefaultConfig.java

@@ -8,8 +8,6 @@ import java.io.Serializable;
 @Data
 public class DetailTypeDefaultConfig implements Serializable {
 
-    private DefaultDisplayInfo defaultDisplayInfo;
-
     // default time to filter tag selection results
     private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.api.pojo;
 
 import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -8,11 +9,11 @@ import lombok.NoArgsConstructor;
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
-public class Dim {
+public class Dimension {
 
     private String name;
 
-    private String type;
+    private DimensionType type;
 
     private String expr;
 
@@ -26,25 +27,15 @@ public class Dim {
 
     private String description;
 
-    private int isTag;
+    public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension) {
-
-    public Dim(String name, String bizName, String type, Integer isCreateDimension) {
         this.name = name;
         this.type = type;
         this.isCreateDimension = isCreateDimension;
         this.bizName = bizName;
     }
 
-    public Dim(String name, String bizName, String type, Integer isCreateDimension, int isTag) {
+    public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension,
-        this.name = name;
+            String expr, String dateFormat, DimensionTimeTypeParams typeParams) {
-        this.type = type;
-        this.isCreateDimension = isCreateDimension;
-        this.bizName = bizName;
-        this.isTag = isTag;
-    }
-
-    public Dim(String name, String type, String expr, String dateFormat,
-            DimensionTimeTypeParams typeParams, Integer isCreateDimension, String bizName) {
         this.name = name;
         this.type = type;
         this.expr = expr;
@@ -54,9 +45,9 @@ public class Dim {
         this.bizName = bizName;
     }
 
-    public static Dim getDefault() {
+    public static Dimension getDefault() {
-        return new Dim("日期", "time", "2023-05-28", Constants.DAY_FORMAT,
+        return new Dimension("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date",
-                new DimensionTimeTypeParams("true", "day"), 0, "imp_date");
+                Constants.DAY_FORMAT, new DimensionTimeTypeParams("false", "day"));
     }
 
     public String getFieldName() {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java

@@ -1,21 +0,0 @@
-package com.tencent.supersonic.headless.api.pojo;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.ToString;
-
-import java.util.List;
-
-@Data
-@ToString
-@AllArgsConstructor
-@NoArgsConstructor
-public class Entity {
-
-    /** uniquely identifies an entity */
-    private Long entityId;
-
-    /** entity name list */
-    private List<String> names;
-}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java

@@ -1,16 +0,0 @@
-package com.tencent.supersonic.headless.api.pojo;
-
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-public class EntityInfo implements Serializable {
-
-    private DataSetInfo dataSetInfo = new DataSetInfo();
-    private List<DataInfo> dimensions = new ArrayList<>();
-    private List<DataInfo> metrics = new ArrayList<>();
-    private String entityId;
-}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java

@@ -18,8 +18,6 @@ public class Identify {
 
     private String bizName;
 
-    private List<String> entityNames;
-
     private Integer isCreateDimension = 0;
 
     public Identify(String name, String type, String bizName) {
@@ -28,13 +26,6 @@ public class Identify {
         this.bizName = bizName;
     }
 
-    public Identify(String name, String type, String bizName, Integer isCreateDimension) {
-        this.name = name;
-        this.type = type;
-        this.bizName = bizName;
-        this.isCreateDimension = isCreateDimension;
-    }
-
     public String getFieldName() {
         return bizName;
     }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java

@@ -1,17 +0,0 @@
-package com.tencent.supersonic.headless.api.pojo;
-
-import com.google.common.collect.Lists;
-import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
-import lombok.Data;
-
-import java.util.List;
-
-@Data
-public class MetricTable {
-
-    private String alias;
-    private List<String> metrics = Lists.newArrayList();
-    private List<String> dimensions = Lists.newArrayList();
-    private String where;
-    private AggOption aggOption = AggOption.DEFAULT;
-}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -24,7 +24,7 @@ public class ModelDetail {
 
     private List<Identify> identifiers = Lists.newArrayList();
 
-    private List<Dim> dimensions = Lists.newArrayList();
+    private List<Dimension> dimensions = Lists.newArrayList();
 
     private List<Measure> measures = Lists.newArrayList();
 
@@ -39,12 +39,11 @@ public class ModelDetail {
         return sqlQuery;
     }
 
-    public List<Dim> filterTimeDims() {
+    public List<Dimension> filterTimeDims() {
         if (CollectionUtils.isEmpty(dimensions)) {
             return Lists.newArrayList();
         }
-        return dimensions.stream()
+        return dimensions.stream().filter(dim -> DimensionType.partition_time.equals(dim.getType()))
-                .filter(dim -> DimensionType.partition_time.name().equalsIgnoreCase(dim.getType()))
                 .collect(Collectors.toList());
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java

@@ -1,5 +1,5 @@
 package com.tencent.supersonic.headless.api.pojo;
 
 public enum SchemaElementType {
-    DATASET, METRIC, DIMENSION, VALUE, ENTITY, ID, DATE, TAG, TERM
+    DATASET, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java

@@ -26,26 +26,27 @@ import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_METRIC_LIMIT;
 public class SemanticParseInfo implements Serializable {
 
     private Integer id;
-    private String queryMode = "PLAIN_TEXT";
+    private String queryMode = "";
-    private SchemaElement dataSet;
     private QueryConfig queryConfig;
+    private QueryType queryType;
+
+    private SchemaElement dataSet;
     private Set<SchemaElement> metrics = Sets.newTreeSet(new SchemaNameLengthComparator());
     private Set<SchemaElement> dimensions = Sets.newTreeSet(new SchemaNameLengthComparator());
-    private SchemaElement entity;
+
-    private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
-    private FilterType filterType = FilterType.AND;
     private Set<QueryFilter> dimensionFilters = Sets.newHashSet();
     private Set<QueryFilter> metricFilters = Sets.newHashSet();
+    private FilterType filterType = FilterType.AND;
+
+    private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
     private Set<Order> orders = Sets.newHashSet();
-    private DateConf dateInfo;
     private long limit = DEFAULT_DETAIL_LIMIT;
     private double score;
     private List<SchemaElementMatch> elementMatches = Lists.newArrayList();
+    private DateConf dateInfo;
     private SqlInfo sqlInfo = new SqlInfo();
-    private SqlEvaluation sqlEvaluation = new SqlEvaluation();
-    private QueryType queryType = QueryType.ID;
-    private EntityInfo entityInfo;
     private String textInfo;
+    private SqlEvaluation sqlEvaluation = new SqlEvaluation();
     private Map<String, Object> properties = Maps.newHashMap();
 
     @Data
@@ -138,8 +139,7 @@ public class SemanticParseInfo implements Serializable {
     public long getDetailLimit() {
         long limit = DEFAULT_DETAIL_LIMIT;
         if (Objects.nonNull(queryConfig)
-                && Objects.nonNull(queryConfig.getDetailTypeDefaultConfig())
+                && Objects.nonNull(queryConfig.getDetailTypeDefaultConfig())) {
-                && Objects.nonNull(queryConfig.getDetailTypeDefaultConfig().getLimit())) {
             limit = queryConfig.getDetailTypeDefaultConfig().getLimit();
         }
         return limit;
@@ -148,8 +148,7 @@ public class SemanticParseInfo implements Serializable {
     public long getMetricLimit() {
         long limit = DEFAULT_METRIC_LIMIT;
         if (Objects.nonNull(queryConfig)
-                && Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig())
+                && Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig())) {
-                && Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig().getLimit())) {
             limit = queryConfig.getAggregateTypeDefaultConfig().getLimit();
         }
         return limit;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java

@@ -13,7 +13,7 @@ import java.util.stream.Collectors;
 
 public class SemanticSchema implements Serializable {
 
-    private List<DataSetSchema> dataSetSchemaList;
+    private final List<DataSetSchema> dataSetSchemaList;
 
     public SemanticSchema(List<DataSetSchema> dataSetSchemaList) {
         this.dataSetSchemaList = dataSetSchemaList;
@@ -27,9 +27,6 @@ public class SemanticSchema implements Serializable {
         Optional<SchemaElement> element = Optional.empty();
 
         switch (elementType) {
-            case ENTITY:
-                element = getElementsById(elementID, getEntities());
-                break;
             case DATASET:
                 element = getElementsById(elementID, getDataSets());
                 break;
@@ -51,11 +48,7 @@ public class SemanticSchema implements Serializable {
             default:
         }
 
-        if (element.isPresent()) {
+        return element.orElse(null);
-            return element.get();
-        } else {
-            return null;
-        }
     }
 
     public Map<Long, String> getDataSetIdToName() {
@@ -65,13 +58,13 @@ public class SemanticSchema implements Serializable {
 
     public List<SchemaElement> getDimensionValues() {
         List<SchemaElement> dimensionValues = new ArrayList<>();
-        dataSetSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues()));
+        dataSetSchemaList.forEach(d -> dimensionValues.addAll(d.getDimensionValues()));
         return dimensionValues;
     }
 
     public List<SchemaElement> getDimensions() {
         List<SchemaElement> dimensions = new ArrayList<>();
-        dataSetSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions()));
+        dataSetSchemaList.forEach(d -> dimensions.addAll(d.getDimensions()));
         return dimensions;
     }
 
@@ -97,26 +90,15 @@ public class SemanticSchema implements Serializable {
         return getElementsByDataSetId(dataSetId, metrics);
     }
 
-    public List<SchemaElement> getEntities() {
-        List<SchemaElement> entities = new ArrayList<>();
-        dataSetSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
-        return entities;
-    }
-
-    public List<SchemaElement> getEntities(Long dataSetId) {
-        List<SchemaElement> entities = getEntities();
-        return getElementsByDataSetId(dataSetId, entities);
-    }
-
     public List<SchemaElement> getTags() {
         List<SchemaElement> tags = new ArrayList<>();
-        dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTags()));
+        dataSetSchemaList.forEach(d -> tags.addAll(d.getTags()));
         return tags;
     }
 
     public List<SchemaElement> getTerms() {
         List<SchemaElement> terms = new ArrayList<>();
-        dataSetSchemaList.stream().forEach(d -> terms.addAll(d.getTerms()));
+        dataSetSchemaList.forEach(d -> terms.addAll(d.getTerms()));
         return terms;
     }
 
@@ -137,22 +119,26 @@ public class SemanticSchema implements Serializable {
         return getElementsById(dataSetId, dataSets).orElse(null);
     }
 
-    public QueryConfig getQueryConfig(Long dataSetId) {
-        DataSetSchema first = dataSetSchemaList.stream().filter(
-                dataSetSchema -> dataSetId.equals(dataSetSchema.getDataSet().getDataSetId()))
-                .findFirst().orElse(null);
-        if (Objects.nonNull(first)) {
-            return first.getQueryConfig();
-        }
-        return null;
-    }
-
     public List<SchemaElement> getDataSets() {
         List<SchemaElement> dataSets = new ArrayList<>();
-        dataSetSchemaList.stream().forEach(d -> dataSets.add(d.getDataSet()));
+        dataSetSchemaList.forEach(d -> dataSets.add(d.getDataSet()));
         return dataSets;
     }
 
+    public DataSetSchema getDataSetSchema(Long dataSetId) {
+        return dataSetSchemaList.stream()
+                .filter(dataSetSchema -> dataSetId.equals(dataSetSchema.getDataSetId())).findFirst()
+                .orElse(null);
+    }
+
+    public QueryConfig getQueryConfig(Long dataSetId) {
+        DataSetSchema dataSetSchema = getDataSetSchema(dataSetId);
+        if (Objects.nonNull(dataSetSchema)) {
+            return dataSetSchema.getQueryConfig();
+        }
+        return null;
+    }
+
     public Map<Long, DataSetSchema> getDataSetSchemaMap() {
         if (CollectionUtils.isEmpty(dataSetSchemaList)) {
             return new HashMap<>();

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java

@@ -45,7 +45,8 @@ public enum DataType {
 
     TDENGINE("TAOS", "TAOS", "com.taosdata.jdbc.TSDBDriver", "'", "'", "\"", "\""),
 
-    POSTGRESQL("postgresql", "postgresql", "org.postgresql.Driver", "'", "'", "\"", "\"");
+    POSTGRESQL("postgresql", "postgresql", "org.postgresql.Driver", "'", "'", "\"", "\""),
+    DUCKDB("duckdb", "duckdb", "org.duckdb.DuckDBDriver", "'", "'", "\"", "\"");
 
     private String feature;
     private String desc;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/FieldType.java

@@ -1,5 +1,5 @@
 package com.tencent.supersonic.headless.api.pojo.enums;
 
 public enum FieldType {
-    primary_key, foreign_key, data_time, dimension, measure;
+    primary_key, foreign_key, partition_time, time, categorical, measure;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java

@@ -32,8 +32,6 @@ public class DimensionReq extends SchemaItem {
 
     private DataTypeEnums dataType;
 
-    private int isTag;
-
     private Map<String, Object> ext;
 
     private DimensionTimeTypeParams typeParams;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
 import com.tencent.supersonic.common.pojo.ChatModelConfig;
+import com.tencent.supersonic.headless.api.pojo.DbSchema;
 import lombok.Data;
 
 import java.util.List;
@@ -8,14 +9,22 @@ import java.util.List;
 @Data
 public class ModelBuildReq {
 
+    private String name;
+
+    private String bizName;
+
     private Long databaseId;
 
+    private Long domainId;
+
     private String sql;
 
     private String db;
 
     private List<String> tables;
 
+    private List<DbSchema> dbSchemas;
+
     private boolean buildByLLM;
 
     private Integer chatModelId;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelReq.java

@@ -35,8 +35,6 @@ public class ModelReq extends SchemaItem {
 
     private List<String> adminOrgs;
 
-    private Long tagObjectId;
-
     private Map<String, Object> ext;
 
     public String getViewer() {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java

@@ -1,24 +0,0 @@
-package com.tencent.supersonic.headless.api.pojo.request;
-
-import com.tencent.supersonic.headless.api.pojo.MetricTable;
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Data
-public class ParseSqlReq {
-    private Map<String, String> variables;
-    private String sql = "";
-    private List<MetricTable> tables;
-    private boolean supportWith = true;
-    private boolean withAlias = true;
-
-    public Map<String, String> getVariables() {
-        if (variables == null) {
-            variables = new HashMap<>();
-        }
-        return variables;
-    }
-}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java

@@ -30,6 +30,6 @@ public class QueryDataSetReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private Long limit = 2000L;
-    private QueryType queryType = QueryType.ID;
+    private QueryType queryType = QueryType.DETAIL;
     private boolean innerLayerNative = false;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java

@@ -34,12 +34,11 @@ public class QueryFilter implements Serializable {
         QueryFilter that = (QueryFilter) o;
         return Objects.equal(bizName, that.bizName) && Objects.equal(name, that.name)
                 && operator == that.operator && Objects.equal(value, that.value)
-                && Objects.equal(elementID, that.elementID)
                 && Objects.equal(function, that.function);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hashCode(bizName, name, operator, value, elementID, function);
+        return Objects.hashCode(bizName, name, operator, value, function);
     }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryNLReq.java

@@ -30,6 +30,7 @@ public class QueryNLReq extends SemanticQueryReq implements Serializable {
     private List<Text2SQLExemplar> dynamicExemplars = Lists.newArrayList();
     private SemanticParseInfo contextParseInfo;
     private SemanticParseInfo selectedParseInfo;
+    private boolean descriptionMapped;
 
     @Override
     public String toCustomizedString() {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.common.util.SqlFilterUtils;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
@@ -42,6 +43,7 @@ import java.util.stream.Collectors;
 @Slf4j
 public class QueryStructReq extends SemanticQueryReq {
 
+    private List<SchemaElement> dimensions = new ArrayList<>();
     private List<String> groups = new ArrayList<>();
     private List<Aggregator> aggregators = new ArrayList<>();
     private List<Order> orders = new ArrayList<>();
@@ -49,7 +51,7 @@ public class QueryStructReq extends SemanticQueryReq {
     private List<Filter> metricFilters = new ArrayList<>();
     private DateConf dateInfo;
     private long limit = Constants.DEFAULT_DETAIL_LIMIT;
-    private QueryType queryType = QueryType.ID;
+    private QueryType queryType = QueryType.DETAIL;
     private boolean convertToSql = true;
 
     public List<String> getGroups() {
@@ -186,19 +188,15 @@ public class QueryStructReq extends SemanticQueryReq {
         List<Aggregator> aggregators = queryStructReq.getAggregators();
         if (!CollectionUtils.isEmpty(aggregators)) {
             for (Aggregator aggregator : aggregators) {
-                selectItems.add(buildAggregatorSelectItem(aggregator, queryStructReq));
+                selectItems.add(buildAggregatorSelectItem(aggregator));
             }
         }
 
         return selectItems;
     }
 
-    private SelectItem buildAggregatorSelectItem(Aggregator aggregator,
+    private SelectItem buildAggregatorSelectItem(Aggregator aggregator) {
-            QueryStructReq queryStructReq) {
         String columnName = aggregator.getColumn();
-        if (queryStructReq.getQueryType().isNativeAggQuery()) {
-            return new SelectItem(new Column(columnName));
-        } else {
         Function function = new Function();
         AggOperatorEnum func = aggregator.getFunc();
         if (AggOperatorEnum.UNKNOWN.equals(func)) {
@@ -211,11 +209,12 @@ public class QueryStructReq extends SemanticQueryReq {
         }
         function.setParameters(new ExpressionList(new Column(columnName)));
         SelectItem selectExpressionItem = new SelectItem(function);
-            String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias()
+        String alias =
-                    : columnName;
+                StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
+        if (!alias.equals(columnName)) {
             selectExpressionItem.setAlias(new Alias(alias));
-            return selectExpressionItem;
         }
+        return selectExpressionItem;
     }
 
     private List<OrderByElement> buildOrderByElements(QueryStructReq queryStructReq) {
@@ -239,7 +238,7 @@ public class QueryStructReq extends SemanticQueryReq {
 
     private GroupByElement buildGroupByElement(QueryStructReq queryStructReq) {
         List<String> groups = queryStructReq.getGroups();
-        if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getQueryType().isNativeAggQuery()) {
+        if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getAggregators().isEmpty()) {
             GroupByElement groupByElement = new GroupByElement();
             for (String group : groups) {
                 groupByElement.addGroupByExpression(new Column(group));

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ValueTaskQueryReq.java

@@ -0,0 +1,23 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import javax.validation.constraints.NotNull;
+
+import com.tencent.supersonic.common.pojo.PageBaseReq;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author: kanedai
+ * @date: 2024/11/24
+ */
+@Data
+public class ValueTaskQueryReq extends PageBaseReq {
+
+    @NotNull
+    private Long itemId;
+
+    private List<String> taskStatusList;
+
+    private String key;
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java

@@ -19,19 +19,4 @@ public class DataSetSchemaResp extends DataSetResp {
     private List<ModelResp> modelResps = Lists.newArrayList();
     private List<TermResp> termResps = Lists.newArrayList();
 
-    public DimSchemaResp getPrimaryKey() {
-        for (ModelResp modelResp : modelResps) {
-            Identify identify = modelResp.getPrimaryIdentify();
-            if (identify == null) {
-                continue;
-            }
-            for (DimSchemaResp dimension : dimensions) {
-                if (identify.getBizName().equals(dimension.getBizName())) {
-                    dimension.setEntityAlias(identify.getEntityNames());
-                    return dimension;
-                }
-            }
-        }
-        return null;
-    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java

@@ -11,5 +11,4 @@ public class DimSchemaResp extends DimensionResp {
 
     private Long useCnt = 0L;
 
-    private List<String> entityAlias;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java

@@ -18,6 +18,8 @@ public class DimensionResp extends SchemaItem {
 
     private Long modelId;
 
+    private Long domainId;
+
     private DimensionType type;
 
     private String expr;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.headless.api.pojo.response;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.Field;
 import com.tencent.supersonic.headless.api.pojo.Identify;
@@ -62,22 +62,7 @@ public class ModelResp extends SchemaItem {
         return isOpen != null && isOpen == 1;
     }
 
-    public Identify getPrimaryIdentify() {
+    public List<Dimension> getTimeDimension() {
-        if (modelDetail == null) {
-            return null;
-        }
-        if (CollectionUtils.isEmpty(modelDetail.getIdentifiers())) {
-            return null;
-        }
-        for (Identify identify : modelDetail.getIdentifiers()) {
-            if (!CollectionUtils.isEmpty(identify.getEntityNames())) {
-                return identify;
-            }
-        }
-        return null;
-    }
-
-    public List<Dim> getTimeDimension() {
         if (modelDetail == null) {
             return Lists.newArrayList();
         }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java

@@ -7,14 +7,11 @@ import com.tencent.supersonic.headless.api.pojo.enums.SchemaType;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
@@ -32,13 +29,6 @@ public class SemanticSchemaResp {
     private DatabaseResp databaseResp;
     private QueryType queryType;
 
-    public String getSchemaKey() {
-        if (dataSetId == null) {
-            return String.format("%s_%s", schemaType, StringUtils.join(modelIds, UNDERLINE));
-        }
-        return String.format("%s_%s", schemaType, dataSetId);
-    }
-
     public MetricSchemaResp getMetric(String bizName) {
         return metrics.stream().filter(metric -> bizName.equalsIgnoreCase(metric.getBizName()))
                 .findFirst().orElse(null);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/ChatQueryContext.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.chat;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.tencent.supersonic.common.pojo.enums.Text2SQLType;
 import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
 import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -41,6 +42,14 @@ public class ChatQueryContext implements Serializable {
         }
     }
 
+    public boolean needSQL() {
+        return !request.getText2SQLType().equals(Text2SQLType.NONE);
+    }
+
+    public DataSetSchema getDataSetSchema(Long dataSetId) {
+        return semanticSchema.getDataSetSchema(dataSetId);
+    }
+
     public List<SemanticQuery> getCandidateQueries() {
         candidateQueries = candidateQueries.stream()
                 .sorted(Comparator.comparing(

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java

@@ -10,7 +10,6 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
 import com.tencent.supersonic.headless.chat.ChatQueryContext;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.util.CollectionUtils;
 
@@ -31,7 +30,8 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
 
     public void correct(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
         try {
-            if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectedS2SQL())) {
+            String s2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
+            if (Objects.isNull(s2SQL)) {
                 return;
             }
             doCorrect(chatQueryContext, semanticParseInfo);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java

@@ -36,8 +36,8 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
             + "\n2.NO NEED to check date filters as the junior engineer seldom makes mistakes in this regard."
             + "\n3.DO NOT miss the AGGREGATE operator of metrics, always add it as needed."
             + "\n4.ALWAYS use `with` statement if nested aggregation is needed."
-            + "\n5.ALWAYS enclose alias created by `AS` command in underscores."
+            + "\n5.ALWAYS enclose alias declared by `AS` command in underscores."
-            + "\n6.ALWAYS translate alias created by `AS` command to the same language as the `#Question`."
+            + "\n6.Alias created by `AS` command must be in the same language ast the `Question`."
             + "\n#Question:{{question}} #InputSQL:{{sql}} #Response:";
 
     public LLMSqlCorrector() {

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java

@@ -1,13 +1,9 @@
 package com.tencent.supersonic.headless.chat.corrector;
 
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
-import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlValidHelper;
-import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.util.ContextUtils;
-import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.chat.ChatQueryContext;
 import lombok.extern.slf4j.Slf4j;
@@ -18,9 +14,7 @@ import org.springframework.util.CollectionUtils;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Objects;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 /** Perform SQL corrections on the "Select" section in S2SQL. */
 @Slf4j
@@ -42,13 +36,11 @@ public class SelectCorrector extends BaseSemanticCorrector {
                 && aggregateFields.size() == selectFields.size()) {
             return;
         }
-        correctS2SQL = addFieldsToSelect(chatQueryContext, semanticParseInfo, correctS2SQL);
+        correctS2SQL = addFieldsToSelect(semanticParseInfo, correctS2SQL);
         semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL);
     }
 
-    protected String addFieldsToSelect(ChatQueryContext chatQueryContext,
+    protected String addFieldsToSelect(SemanticParseInfo semanticParseInfo, String correctS2SQL) {
-            SemanticParseInfo semanticParseInfo, String correctS2SQL) {
-        correctS2SQL = addTagDefaultFields(chatQueryContext, semanticParseInfo, correctS2SQL);
 
         Set<String> selectFields = new HashSet<>(SqlSelectHelper.getSelectFields(correctS2SQL));
         Set<String> needAddFields = new HashSet<>(SqlSelectHelper.getGroupByFields(correctS2SQL));
@@ -70,34 +62,4 @@ public class SelectCorrector extends BaseSemanticCorrector {
         return addFieldsToSelectSql;
     }
 
-    private String addTagDefaultFields(ChatQueryContext chatQueryContext,
-            SemanticParseInfo semanticParseInfo, String correctS2SQL) {
-        // If it is in DETAIL mode and select *, add default metrics and dimensions.
-        boolean hasAsterisk = SqlSelectFunctionHelper.hasAsterisk(correctS2SQL);
-        if (!(hasAsterisk && QueryType.DETAIL.equals(semanticParseInfo.getQueryType()))) {
-            return correctS2SQL;
-        }
-        Long dataSetId = semanticParseInfo.getDataSetId();
-        DataSetSchema dataSetSchema =
-                chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
-        Set<String> needAddDefaultFields = new HashSet<>();
-        if (Objects.nonNull(dataSetSchema)) {
-            if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultMetrics())) {
-                Set<String> metrics = dataSetSchema.getTagDefaultMetrics().stream()
-                        .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
-                needAddDefaultFields.addAll(metrics);
-            }
-            if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultDimensions())) {
-                Set<String> dimensions = dataSetSchema.getTagDefaultDimensions().stream()
-                        .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
-                needAddDefaultFields.addAll(dimensions);
-            }
-        }
-        // remove * in sql and add default fields.
-        if (!CollectionUtils.isEmpty(needAddDefaultFields)) {
-            correctS2SQL =
-                    SqlRemoveHelper.removeAsteriskAndAddFields(correctS2SQL, needAddDefaultFields);
-        }
-        return correctS2SQL;
-    }
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java

@@ -19,6 +19,7 @@ public class DictWord {
     private String word;
     private String nature;
     private String natureWithFrequency;
+    private String alias;
 
     @Override
     public boolean equals(Object o) {