From e55f43c7376b3c44a1248b46022176daa0e1534e Mon Sep 17 00:00:00 2001 From: Jun Zhang Date: Sun, 8 Dec 2024 13:32:29 +0800 Subject: [PATCH] [improvement]Use `QueryWrapper` in place of hard-coded SQLs (#1944) * [improvement][launcher]Use API to get element ID avoiding hard-code. * [fix][launcher]Fix mysql scripts. * [improvement][launcher]Support DuckDB database and refactor translator code structure. * [improvement][headless-fe] Revamped the interaction for semantic modeling routing and successfully implemented the switching between dimension and dataset management. * [improvement][Headless] Add table ddl in Dbschema * [improvement][Headless] Add get database by type * [improvement][Headless] Supports automatic batch creation of models based on db table names. * [improvement][Headless] Supports getting domain by bizName * [improvement][launcher]Refactor unit tests and demo data. * [fix][launcher]Change default vector dimension to 512. * [improvement](Dict) add dimValueAliasMap info for KnowledgeBaseService * [improvement][headless]Use QueryWrapper to replace hard-code SQL in mapper xml. * [improvement][chat]Introduce ChatMemory to delegate ChatMemoryDO. * [fix][common]Fix embedding store sys configs. * [fix][common]Fix postgres schema, using varchar instead of char. * [improvement][launcher]Change supersonic docker deployment from mysql to postgres. * [Fix][launcher]Fix a number of issues related to semantic modeling. * [Fix][headless]Fix the evaluation logic of agg type. * [fix][assembly]Fix Dockerfile and add docker compose run script. * [fix][chat]Fix "multiple assignments to same column "similar_queries". * [improvement][headless]Use LamdaQueryWrapper to avoid hard-coded column names. * [improvement][headless]Refactor headless infra to support advanced semantic modelling. * [improvement][headless]Change class name `Dim` to `Dimension`. * [improvement][chat]Introduce `TimeFieldMapper` to always map time field. * [fix][headless]Remove unnecessary dimension existence check. * [fix][chat]Fix adjusted filters don't take effect. --------- --- .../repository/impl/UserRepositoryImpl.java | 4 +- .../api/pojo/request/ChatMemoryUpdateReq.java | 2 + .../chat/server/executor/SqlExecutor.java | 4 +- .../chat/server/memory/MemoryReviewTask.java | 25 +- .../chat/server/parser/NL2SQLParser.java | 6 + .../persistence/dataobject/ChatMemoryDO.java | 14 +- .../impl/ChatQueryRepositoryImpl.java | 1 - .../chat/server/pojo/ChatMemory.java | 48 + .../parse/QueryRecommendProcessor.java | 3 +- .../chat/server/rest/MemoryController.java | 6 +- .../chat/server/service/MemoryService.java | 15 +- .../server/service/impl/AgentServiceImpl.java | 4 +- .../service/impl/ChatQueryServiceImpl.java | 64 +- .../service/impl/MemoryServiceImpl.java | 77 +- .../com/hankcs/hanlp/seg/common/Term.java | 13 - .../config/EmbeddingStoreParameterConfig.java | 26 +- .../supersonic/common/pojo/DataItem.java | 4 +- .../supersonic/common/pojo/ModelRela.java | 3 +- .../common/pojo/enums/TypeEnums.java | 2 +- .../spring/PgvectorEmbeddingStoreFactory.java | 2 +- .../store/embedding/TextSegmentConvert.java | 17 +- docker/Dockerfile | 13 +- docker/DockerfileS2 | 32 + docker/docker-compose-run.sh | 3 + docker/docker-compose.yml | 80 +- .../headless/api/pojo/ColumnSchema.java | 2 +- .../api/pojo/{Dim.java => Dimension.java} | 12 +- .../headless/api/pojo/ModelDetail.java | 4 +- .../headless/api/pojo/enums/FieldType.java | 2 +- .../api/pojo/request/DimensionReq.java | 2 - .../api/pojo/request/ModelBuildReq.java | 4 + .../api/pojo/request/QueryStructReq.java | 4 +- .../api/pojo/response/DimensionResp.java | 2 + .../headless/api/pojo/response/ModelResp.java | 4 +- .../headless/chat/knowledge/DictWord.java | 1 + .../chat/knowledge/KnowledgeBaseService.java | 33 + .../chat/knowledge/MultiCustomDictionary.java | 19 +- .../headless/chat/mapper/KeywordMapper.java | 25 +- .../headless/chat/mapper/TimeFieldMapper.java | 37 + .../core/adaptor/db/BaseDbAdaptor.java | 4 +- .../headless/core/adaptor/db/DbAdaptor.java | 3 + .../core/adaptor/db/PostgresqlAdaptor.java | 5 +- .../converter/SqlQueryConverter.java | 11 +- .../converter/StructQueryConverter.java | 6 +- .../parser/calcite/node/DataModelNode.java | 54 +- .../parser/s2sql/OntologyQueryParam.java | 4 +- .../core/utils/SysTimeDimensionBuilder.java | 27 +- .../service/impl/S2SemanticLayerService.java | 2 +- .../listener/SchemaDictUpdateListener.java | 2 +- .../server/manager/ModelYamlManager.java | 16 +- .../server/modeller/RuleSemanticModeller.java | 2 +- .../persistence/dataobject/ModelDO.java | 1 + .../persistence/dataobject/QueryStatDO.java | 1 + .../mapper/DimensionDOCustomMapper.java | 5 - .../mapper/MetricDOCustomMapper.java | 3 - .../server/persistence/mapper/StatMapper.java | 6 - .../repository/impl/DictRepositoryImpl.java | 1 + .../impl/DimensionRepositoryImpl.java | 41 +- .../repository/impl/MetricRepositoryImpl.java | 43 +- .../impl/QueryRuleRepositoryImpl.java | 2 +- .../repository/impl/StatRepositoryImpl.java | 18 +- .../server/service/DimensionService.java | 2 +- .../service/impl/DataSetServiceImpl.java | 4 +- .../service/impl/DatabaseServiceImpl.java | 3 + .../service/impl/DimensionServiceImpl.java | 45 +- .../service/impl/MetricServiceImpl.java | 71 +- .../server/service/impl/ModelServiceImpl.java | 39 +- .../service/impl/TagQueryServiceImpl.java | 10 +- .../server/task/MetaEmbeddingTask.java | 2 +- .../headless/server/utils/DictUtils.java | 8 +- .../server/utils/DimensionConverter.java | 2 + .../server/utils/MetricConverter.java | 4 +- .../headless/server/utils/ModelConverter.java | 23 +- .../src/main/resources/mapper/StatMapper.xml | 24 - .../mapper/custom/DimensionDOCustomMapper.xml | 57 - .../mapper/custom/MetricDOCustomMapper.xml | 64 - .../server/service/ModelServiceImplTest.java | 32 +- .../main/resources/META-INF/spring.factories | 5 +- .../supersonic/demo/S2CompanyDemo.java | 25 +- .../tencent/supersonic/demo/S2SingerDemo.java | 10 +- .../tencent/supersonic/demo/S2VisitsDemo.java | 20 +- .../main/resources/META-INF/spring.factories | 5 +- .../main/resources/application-docker.yaml | 26 + .../main/resources/application-postgres.yaml | 14 +- .../custom/dic_value_1_DIMENSION_2.txt | 6 - .../custom/dic_value_3_DIMENSION_4.txt | 4 - .../custom/dic_value_4_DIMENSION_5.txt | 3 - .../custom/dic_value_4_DIMENSION_6.txt | 6 - .../custom/dic_value_4_DIMENSION_7.txt | 2 - .../custom/dic_value_4_DIMENSION_8.txt | 6 - .../src/main/resources/db/data-mysql-demo.sql | 106 +- .../main/resources/db/data-postgres-demo.sql | 66 +- .../src/main/resources/db/schema-postgres.sql | 6 +- .../headless/SemanticModellerTest.java | 8 +- .../src/test/resources/data/README.url | 2 - .../dictionary/CoreNatureDictionary.mini.txt | 3 - .../CoreNatureDictionary.ngram.mini.txt | 4 - .../data/dictionary/custom/DimValue_10_20.txt | 8 - .../data/dictionary/custom/DimValue_10_22.txt | 8 - .../data/dictionary/custom/DimValue_1_1.txt | 5 - .../data/dictionary/custom/DimValue_1_2.txt | 18 - .../data/dictionary/custom/DimValue_1_3.txt | 6 - .../data/dictionary/custom/DimValue_4_8.txt | 9 - .../data/dictionary/custom/DimValue_5_10.txt | 4 - .../data/dictionary/custom/DimValue_5_11.txt | 6 - .../data/dictionary/custom/DimValue_6_12.txt | 4 - .../data/dictionary/custom/DimValue_6_13.txt | 2 - .../data/dictionary/custom/DimValue_7_16.txt | 2 - .../data/dictionary/custom/DimValue_8_18.txt | 4 - .../data/dictionary/custom/DimValue_8_19.txt | 2 - .../data/dictionary/custom/DimValue_8_21.txt | 6 - .../data/dictionary/custom/DimValue_9_15.txt | 5 - .../data/dictionary/custom/DimValue_9_16.txt | 4 - .../data/dictionary/custom/DimValue_9_18.txt | 7 - .../data/dictionary/custom/DimValue_9_19.txt | 7 - .../data/dictionary/other/CharTable.txt | 4890 ----------------- .../data/dictionary/other/TagPKU98.csv | 44 - .../src/test/resources/data/version.txt | 1 - .../src/test/resources/hanlp.properties | 2 - .../src/components/ChatMsg/Table/index.tsx | 17 +- 120 files changed, 844 insertions(+), 5810 deletions(-) create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatMemory.java create mode 100644 docker/DockerfileS2 create mode 100644 docker/docker-compose-run.sh rename headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/{Dim.java => Dimension.java} (77%) create mode 100644 headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TimeFieldMapper.java create mode 100644 launchers/standalone/src/main/resources/application-docker.yaml delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/dic_value_1_DIMENSION_2.txt delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/dic_value_3_DIMENSION_4.txt delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/dic_value_4_DIMENSION_5.txt delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/dic_value_4_DIMENSION_6.txt delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/dic_value_4_DIMENSION_7.txt delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/dic_value_4_DIMENSION_8.txt delete mode 100644 launchers/standalone/src/test/resources/data/README.url delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/CoreNatureDictionary.mini.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/CoreNatureDictionary.ngram.mini.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_20.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_22.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_1.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_8.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_15.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_16.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_18.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_19.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/other/CharTable.txt delete mode 100644 launchers/standalone/src/test/resources/data/dictionary/other/TagPKU98.csv delete mode 100644 launchers/standalone/src/test/resources/data/version.txt delete mode 100644 launchers/standalone/src/test/resources/hanlp.properties diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java index 1549a1db6..07ab94c75 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java @@ -51,7 +51,7 @@ public class UserRepositoryImpl implements UserRepository { @Override public List getUserTokenListByName(String userName) { QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("user_name", userName); + queryWrapper.lambda().eq(UserTokenDO::getUserName, userName); return userTokenDOMapper.selectList(queryWrapper); } @@ -68,7 +68,7 @@ public class UserRepositoryImpl implements UserRepository { @Override public void deleteUserTokenByName(String userName) { QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("user_name", userName); + queryWrapper.lambda().eq(UserTokenDO::getUserName, userName); userTokenDOMapper.delete(queryWrapper); } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java index 39a7f3c28..9b68c10e5 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatMemoryUpdateReq.java @@ -4,9 +4,11 @@ import javax.validation.constraints.NotNull; import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; +import lombok.Builder; import lombok.Data; @Data +@Builder public class ChatMemoryUpdateReq { @NotNull(message = "id不可为空") diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java index a77d8c03f..2c3644b3a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.executor; import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; import com.tencent.supersonic.chat.server.pojo.ChatContext; +import com.tencent.supersonic.chat.server.pojo.ChatMemory; import com.tencent.supersonic.chat.server.pojo.ExecuteContext; import com.tencent.supersonic.chat.server.service.ChatContextService; import com.tencent.supersonic.chat.server.service.MemoryService; @@ -44,7 +44,7 @@ public class SqlExecutor implements ChatQueryExecutor { Text2SQLExemplar.class); MemoryService memoryService = ContextUtils.getBean(MemoryService.class); - memoryService.createMemory(ChatMemoryDO.builder() + memoryService.createMemory(ChatMemory.builder() .agentId(executeContext.getAgent().getId()).status(MemoryStatus.PENDING) .question(exemplar.getQuestion()).sideInfo(exemplar.getSideInfo()) .dbSchema(exemplar.getDbSchema()).s2sql(exemplar.getSql()) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java index 09889e489..30c81460a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.chat.server.memory; import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; +import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter; import com.tencent.supersonic.chat.server.agent.Agent; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; +import com.tencent.supersonic.chat.server.pojo.ChatMemory; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.MemoryService; import com.tencent.supersonic.common.pojo.ChatApp; @@ -66,7 +67,7 @@ public class MemoryReviewTask { } ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId()).build(); - memoryService.getMemories(chatMemoryFilter).stream().forEach(memory -> { + memoryService.getMemories(chatMemoryFilter).forEach(memory -> { try { processMemory(memory, agent); } catch (Exception e) { @@ -77,23 +78,19 @@ public class MemoryReviewTask { } } - private void processMemory(ChatMemoryDO m, Agent agent) { + private void processMemory(ChatMemory m, Agent agent) { if (Objects.isNull(agent)) { log.warn("Agent id {} not found or memory review disabled", m.getAgentId()); return; } - ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY); - if (Objects.isNull(chatApp) || !chatApp.isEnable()) { + // if either LLM or human has reviewed, just return + if (Objects.nonNull(m.getLlmReviewRet()) || Objects.nonNull(m.getHumanReviewRet())) { return; } - // 如果大模型已经评估过，则不再评估 - if (Objects.nonNull(m.getLlmReviewRet())) { - // directly enable memory if the LLM determines it positive - if (MemoryReviewResult.POSITIVE.equals(m.getLlmReviewRet())) { - memoryService.enableMemory(m); - } + ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY); + if (Objects.isNull(chatApp) || !chatApp.isEnable()) { return; } @@ -112,19 +109,19 @@ public class MemoryReviewTask { } } - private String createPromptString(ChatMemoryDO m, String promptTemplate) { + private String createPromptString(ChatMemory m, String promptTemplate) { return String.format(promptTemplate, m.getQuestion(), m.getDbSchema(), m.getSideInfo(), m.getS2sql()); } - private void processResponse(String response, ChatMemoryDO m) { + private void processResponse(String response, ChatMemory m) { Matcher matcher = OUTPUT_PATTERN.matcher(response); if (matcher.find()) { m.setLlmReviewRet(MemoryReviewResult.getMemoryReviewResult(matcher.group(1))); m.setLlmReviewCmt(matcher.group(2)); // directly enable memory if the LLM determines it positive if (MemoryReviewResult.POSITIVE.equals(m.getLlmReviewRet())) { - memoryService.enableMemory(m); + m.setStatus(MemoryStatus.ENABLED); } memoryService.updateMemory(m); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java index de06afcaf..d82108bd0 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java @@ -91,6 +91,7 @@ public class NL2SQLParser implements ChatQueryParser { // mapModes Set requestedDatasets = queryNLReq.getDataSetIds(); List candidateParses = Lists.newArrayList(); + StringBuilder errMsg = new StringBuilder(); for (Long datasetId : requestedDatasets) { queryNLReq.setDataSetIds(Collections.singleton(datasetId)); ChatParseResp parseResp = new ChatParseResp(parseContext.getRequest().getQueryId()); @@ -104,6 +105,7 @@ public class NL2SQLParser implements ChatQueryParser { doParse(queryNLReq, parseResp); } if (parseResp.getSelectedParses().isEmpty()) { + errMsg.append(parseResp.getErrorMsg()); continue; } // for one dataset select the top 1 parse after sorting @@ -116,6 +118,10 @@ public class NL2SQLParser implements ChatQueryParser { SemanticParseInfo.sort(candidateParses); parseContext.getResponse().setSelectedParses( candidateParses.subList(0, Math.min(parserShowCount, candidateParses.size()))); + if (parseContext.getResponse().getSelectedParses().isEmpty()) { + parseContext.getResponse().setState(ParseResp.ParseState.FAILED); + parseContext.getResponse().setErrorMsg(errMsg.toString()); + } } // next go with llm-based parsers unless LLM is disabled or use feedback is needed. diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java index d03ca14b8..7622f53fd 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatMemoryDO.java @@ -4,17 +4,17 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; -import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import lombok.ToString; +import lombok.NoArgsConstructor; import java.util.Date; @Data @Builder -@ToString +@NoArgsConstructor +@AllArgsConstructor @TableName("s2_chat_memory") public class ChatMemoryDO { @TableId(type = IdType.AUTO) @@ -36,16 +36,16 @@ public class ChatMemoryDO { private String s2sql; @TableField("status") - private MemoryStatus status; + private String status; @TableField("llm_review") - private MemoryReviewResult llmReviewRet; + private String llmReviewRet; @TableField("llm_comment") private String llmReviewCmt; @TableField("human_review") - private MemoryReviewResult humanReviewRet; + private String humanReviewRet; @TableField("human_comment") private String humanReviewCmt; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java index 3148b3118..e934dc599 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java @@ -20,7 +20,6 @@ import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryReposi import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.PageUtils; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatMemory.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatMemory.java new file mode 100644 index 000000000..16227be13 --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/pojo/ChatMemory.java @@ -0,0 +1,48 @@ +package com.tencent.supersonic.chat.server.pojo; + +import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; +import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.Date; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@ToString +public class ChatMemory { + private Long id; + + private Integer agentId; + + private String question; + + private String sideInfo; + + private String dbSchema; + + private String s2sql; + + private MemoryStatus status; + + private MemoryReviewResult llmReviewRet; + + private String llmReviewCmt; + + private MemoryReviewResult humanReviewRet; + + private String humanReviewCmt; + + private String createdBy; + + private Date createdAt; + + private String updatedBy; + + private Date updatedAt; +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java index 8a0ac855e..b060d0055 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java @@ -56,8 +56,7 @@ public class QueryRecommendProcessor implements ParseResultProcessor { private void updateChatQuery(ChatQueryDO chatQueryDO) { ChatQueryRepository chatQueryRepository = ContextUtils.getBean(ChatQueryRepository.class); UpdateWrapper updateWrapper = new UpdateWrapper<>(); - updateWrapper.eq("question_id", chatQueryDO.getQuestionId()); - updateWrapper.set("similar_queries", chatQueryDO.getSimilarQueries()); + updateWrapper.lambda().eq(ChatQueryDO::getQuestionId, chatQueryDO.getQuestionId()); chatQueryRepository.updateChatQuery(chatQueryDO, updateWrapper); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java index 10d4bdea6..ab9c19b6b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/MemoryController.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryCreateReq; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq; import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; +import com.tencent.supersonic.chat.server.pojo.ChatMemory; import com.tencent.supersonic.chat.server.service.MemoryService; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; @@ -32,7 +32,7 @@ public class MemoryController { public Boolean createMemory(@RequestBody ChatMemoryCreateReq chatMemoryCreateReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - memoryService.createMemory(ChatMemoryDO.builder().agentId(chatMemoryCreateReq.getAgentId()) + memoryService.createMemory(ChatMemory.builder().agentId(chatMemoryCreateReq.getAgentId()) .s2sql(chatMemoryCreateReq.getS2sql()).question(chatMemoryCreateReq.getQuestion()) .dbSchema(chatMemoryCreateReq.getDbSchema()).status(chatMemoryCreateReq.getStatus()) .humanReviewRet(MemoryReviewResult.POSITIVE).createdBy(user.getName()) @@ -49,7 +49,7 @@ public class MemoryController { } @RequestMapping("/pageMemories") - public PageInfo pageMemories(@RequestBody PageMemoryReq pageMemoryReq) { + public PageInfo pageMemories(@RequestBody PageMemoryReq pageMemoryReq) { return memoryService.pageMemories(pageMemoryReq); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/MemoryService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/MemoryService.java index 6189ae386..3747f7fd1 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/MemoryService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/MemoryService.java @@ -4,27 +4,22 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq; import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; +import com.tencent.supersonic.chat.server.pojo.ChatMemory; import com.tencent.supersonic.common.pojo.User; import java.util.List; public interface MemoryService { - void createMemory(ChatMemoryDO memory); + void createMemory(ChatMemory memory); void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user); - void updateMemory(ChatMemoryDO memory); - - void enableMemory(ChatMemoryDO memory); - - void disableMemory(ChatMemoryDO memory); + void updateMemory(ChatMemory memory); void batchDelete(List ids); - PageInfo pageMemories(PageMemoryReq pageMemoryReq); + PageInfo pageMemories(PageMemoryReq pageMemoryReq); - List getMemories(ChatMemoryFilter chatMemoryFilter); + List getMemories(ChatMemoryFilter chatMemoryFilter); - List getMemoriesForLlmReview(); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index 591d3a21b..e8e8ea012 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -6,8 +6,8 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.agent.VisualConfig; import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; import com.tencent.supersonic.chat.server.persistence.mapper.AgentDOMapper; +import com.tencent.supersonic.chat.server.pojo.ChatMemory; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatQueryService; import com.tencent.supersonic.chat.server.service.MemoryService; @@ -121,7 +121,7 @@ public class AgentServiceImpl extends ServiceImpl implem ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId()).questions(examples).build(); List memoriesExisted = memoryService.getMemories(chatMemoryFilter).stream() - .map(ChatMemoryDO::getQuestion).collect(Collectors.toList()); + .map(ChatMemory::getQuestion).collect(Collectors.toList()); for (String example : examples) { if (memoriesExisted.contains(example)) { continue; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java index 9c95d1fc3..92a1b6a4d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java @@ -18,11 +18,7 @@ import com.tencent.supersonic.chat.server.service.ChatManageService; import com.tencent.supersonic.chat.server.service.ChatQueryService; import com.tencent.supersonic.chat.server.util.ComponentFactory; import com.tencent.supersonic.chat.server.util.QueryReqConverter; -import com.tencent.supersonic.common.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; -import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; -import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.jsqlparser.*; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.util.DateUtils; @@ -48,11 +44,7 @@ import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; -import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; -import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; -import net.sf.jsqlparser.expression.operators.relational.InExpression; -import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; -import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; +import net.sf.jsqlparser.expression.operators.relational.*; import net.sf.jsqlparser.schema.Column; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -60,14 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -210,20 +195,22 @@ public class ChatQueryServiceImpl implements ChatQueryService { private void handleLLMQueryMode(ChatQueryDataReq chatQueryDataReq, SemanticQuery semanticQuery, DataSetSchema dataSetSchema, User user) throws Exception { SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); - List fields = getFieldsFromSql(parseInfo); - if (checkMetricReplace(fields, chatQueryDataReq.getMetrics())) { - log.info("llm begin replace metrics!"); + String rebuiltS2SQL; + if (checkMetricReplace(chatQueryDataReq, parseInfo)) { + log.info("rebuild S2SQL with adjusted metrics!"); SchemaElement metricToReplace = chatQueryDataReq.getMetrics().iterator().next(); - replaceMetrics(parseInfo, metricToReplace); + rebuiltS2SQL = replaceMetrics(parseInfo, metricToReplace); } else { - log.info("llm begin revise filters!"); - String correctorSql = reviseCorrectS2SQL(chatQueryDataReq, parseInfo, dataSetSchema); - parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql); - semanticQuery.setParseInfo(parseInfo); - SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); - SemanticTranslateResp explain = semanticLayerService.translate(semanticQueryReq, user); - parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL()); + log.info("rebuild S2SQL with adjusted filters!"); + rebuiltS2SQL = replaceFilters(chatQueryDataReq, parseInfo, dataSetSchema); } + // reset SqlInfo and request re-translation + parseInfo.getSqlInfo().setCorrectedS2SQL(rebuiltS2SQL); + parseInfo.getSqlInfo().setParsedS2SQL(rebuiltS2SQL); + parseInfo.getSqlInfo().setQuerySQL(null); + SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); + SemanticTranslateResp explain = semanticLayerService.translate(semanticQueryReq, user); + parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL()); } private void handleRuleQueryMode(SemanticQuery semanticQuery, DataSetSchema dataSetSchema, @@ -243,7 +230,9 @@ public class ChatQueryServiceImpl implements ChatQueryService { return queryResult; } - private boolean checkMetricReplace(List oriFields, Set metrics) { + private boolean checkMetricReplace(ChatQueryDataReq chatQueryDataReq, SemanticParseInfo parseInfo) { + List oriFields = getFieldsFromSql(parseInfo); + Set metrics = chatQueryDataReq.getMetrics(); if (CollectionUtils.isEmpty(oriFields) || CollectionUtils.isEmpty(metrics)) { return false; } @@ -252,8 +241,8 @@ public class ChatQueryServiceImpl implements ChatQueryService { return !oriFields.containsAll(metricNames); } - private String reviseCorrectS2SQL(ChatQueryDataReq queryData, SemanticParseInfo parseInfo, - DataSetSchema dataSetSchema) { + private String replaceFilters(ChatQueryDataReq queryData, SemanticParseInfo parseInfo, + DataSetSchema dataSetSchema) { String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL(); log.info("correctorSql before replacing:{}", correctorSql); // get where filter and having filter @@ -290,7 +279,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { return correctorSql; } - private void replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) { + private String replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) { List oriMetrics = parseInfo.getMetrics().stream().map(SchemaElement::getName) .collect(Collectors.toList()); String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL(); @@ -302,7 +291,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { correctorSql = SqlReplaceHelper.replaceAggFields(correctorSql, fieldMap); } log.info("after replaceMetrics:{}", correctorSql); - parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql); + return correctorSql; } private QueryResult doExecution(SemanticQueryReq semanticQueryReq, String queryMode, User user) @@ -477,6 +466,9 @@ public class ChatQueryServiceImpl implements ChatQueryService { } private void mergeParseInfo(SemanticParseInfo parseInfo, ChatQueryDataReq queryData) { + if (Objects.nonNull(queryData.getDateInfo())) { + parseInfo.setDateInfo(queryData.getDateInfo()); + } if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { return; } @@ -492,9 +484,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { if (!CollectionUtils.isEmpty(queryData.getMetricFilters())) { parseInfo.setMetricFilters(queryData.getMetricFilters()); } - if (Objects.nonNull(queryData.getDateInfo())) { - parseInfo.setDateInfo(queryData.getDateInfo()); - } + parseInfo.setSqlInfo(new SqlInfo()); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java index 4ab32b8dc..288cc5978 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java @@ -3,12 +3,14 @@ package com.tencent.supersonic.chat.server.service.impl; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; +import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult; import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq; import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO; import com.tencent.supersonic.chat.server.persistence.repository.ChatMemoryRepository; +import com.tencent.supersonic.chat.server.pojo.ChatMemory; import com.tencent.supersonic.chat.server.service.MemoryService; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.pojo.Text2SQLExemplar; @@ -16,12 +18,15 @@ import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.service.ExemplarService; import com.tencent.supersonic.common.util.BeanMapper; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import java.util.Date; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; @Service public class MemoryServiceImpl implements MemoryService { @@ -36,20 +41,22 @@ public class MemoryServiceImpl implements MemoryService { private EmbeddingConfig embeddingConfig; @Override - public void createMemory(ChatMemoryDO memory) { + public void createMemory(ChatMemory memory) { // if an existing enabled memory has the same question, just skip - List memories = + List memories = getMemories(ChatMemoryFilter.builder().agentId(memory.getAgentId()) .question(memory.getQuestion()).status(MemoryStatus.ENABLED).build()); - if (memories.size() == 0) { - chatMemoryRepository.createMemory(memory); + if (memories.isEmpty()) { + ChatMemoryDO memoryDO = getMemoryDO(memory); + chatMemoryRepository.createMemory(memoryDO); } } @Override public void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user) { ChatMemoryDO chatMemoryDO = chatMemoryRepository.getMemory(chatMemoryUpdateReq.getId()); - boolean hadEnabled = MemoryStatus.ENABLED.equals(chatMemoryDO.getStatus()); + boolean hadEnabled = + MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim()); chatMemoryDO.setUpdatedBy(user.getName()); chatMemoryDO.setUpdatedAt(new Date()); BeanMapper.mapper(chatMemoryUpdateReq, chatMemoryDO); @@ -58,12 +65,12 @@ public class MemoryServiceImpl implements MemoryService { } else if (MemoryStatus.DISABLED.equals(chatMemoryUpdateReq.getStatus()) && hadEnabled) { disableMemory(chatMemoryDO); } - updateMemory(chatMemoryDO); + chatMemoryRepository.updateMemory(chatMemoryDO); } @Override - public void updateMemory(ChatMemoryDO memory) { - chatMemoryRepository.updateMemory(memory); + public void updateMemory(ChatMemory memory) { + chatMemoryRepository.updateMemory(getMemoryDO(memory)); } @Override @@ -72,7 +79,7 @@ public class MemoryServiceImpl implements MemoryService { } @Override - public PageInfo pageMemories(PageMemoryReq pageMemoryReq) { + public PageInfo pageMemories(PageMemoryReq pageMemoryReq) { ChatMemoryFilter chatMemoryFilter = pageMemoryReq.getChatMemoryFilter(); chatMemoryFilter.setSort(pageMemoryReq.getSort()); chatMemoryFilter.setOrderCondition(pageMemoryReq.getOrderCondition()); @@ -81,7 +88,7 @@ public class MemoryServiceImpl implements MemoryService { } @Override - public List getMemories(ChatMemoryFilter chatMemoryFilter) { + public List getMemories(ChatMemoryFilter chatMemoryFilter) { QueryWrapper queryWrapper = new QueryWrapper<>(); if (chatMemoryFilter.getAgentId() != null) { queryWrapper.lambda().eq(ChatMemoryDO::getAgentId, chatMemoryFilter.getAgentId()); @@ -109,32 +116,52 @@ public class MemoryServiceImpl implements MemoryService { queryWrapper.orderBy(true, chatMemoryFilter.isAsc(), chatMemoryFilter.getOrderCondition()); } - return chatMemoryRepository.getMemories(queryWrapper); + List chatMemoryDOS = chatMemoryRepository.getMemories(queryWrapper); + return chatMemoryDOS.stream().map(this::getMemory).collect(Collectors.toList()); } - @Override - public List getMemoriesForLlmReview() { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING) - .isNull(ChatMemoryDO::getLlmReviewRet); - return chatMemoryRepository.getMemories(queryWrapper); - } - - @Override - public void enableMemory(ChatMemoryDO memory) { - memory.setStatus(MemoryStatus.ENABLED); + private void enableMemory(ChatMemoryDO memory) { + memory.setStatus(MemoryStatus.ENABLED.toString()); exemplarService.storeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()), Text2SQLExemplar.builder().question(memory.getQuestion()) .sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema()) .sql(memory.getS2sql()).build()); } - @Override - public void disableMemory(ChatMemoryDO memory) { - memory.setStatus(MemoryStatus.DISABLED); + private void disableMemory(ChatMemoryDO memory) { + memory.setStatus(MemoryStatus.DISABLED.toString()); exemplarService.removeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()), Text2SQLExemplar.builder().question(memory.getQuestion()) .sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema()) .sql(memory.getS2sql()).build()); } + + private ChatMemoryDO getMemoryDO(ChatMemory memory) { + ChatMemoryDO memoryDO = new ChatMemoryDO(); + BeanUtils.copyProperties(memory, memoryDO); + memoryDO.setStatus(memory.getStatus().toString().trim()); + if (Objects.nonNull(memory.getHumanReviewRet())) { + memoryDO.setHumanReviewRet(memory.getHumanReviewRet().toString().trim()); + } + if (Objects.nonNull(memory.getLlmReviewRet())) { + memoryDO.setLlmReviewRet(memory.getLlmReviewRet().toString().trim()); + } + + return memoryDO; + } + + private ChatMemory getMemory(ChatMemoryDO memoryDO) { + ChatMemory memory = new ChatMemory(); + BeanUtils.copyProperties(memoryDO, memory); + memory.setStatus(MemoryStatus.valueOf(memoryDO.getStatus().trim())); + if (Objects.nonNull(memoryDO.getHumanReviewRet())) { + memory.setHumanReviewRet( + MemoryReviewResult.valueOf(memoryDO.getHumanReviewRet().trim())); + } + if (Objects.nonNull(memoryDO.getLlmReviewRet())) { + memory.setLlmReviewRet(MemoryReviewResult.valueOf(memoryDO.getLlmReviewRet().trim())); + } + return memory; + } + } diff --git a/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java b/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java index 8bfd2f5e0..89c77565b 100644 --- a/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java +++ b/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java @@ -19,19 +19,6 @@ public class Term { this.nature = nature; } - public Term(String word, Nature nature, int offset) { - this.word = word; - this.nature = nature; - this.offset = offset; - } - - public Term(String word, Nature nature, int offset, int frequency) { - this.word = word; - this.nature = nature; - this.offset = offset; - this.frequency = frequency; - } - public int length() { return this.word.length(); } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java index 69a6b7ae4..5591e63cb 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java @@ -38,14 +38,14 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", MODULE_NAME); public static final Parameter EMBEDDING_STORE_DIMENSION = - new Parameter("s2.embedding.store.dimension", "", "纬度", "", "number", MODULE_NAME, null, - getDimensionDependency()); + new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME, + null, getDimensionDependency()); public static final Parameter EMBEDDING_STORE_DATABASE_NAME = new Parameter("s2.embedding.store.databaseName", "", "DatabaseName", "", "string", MODULE_NAME, null, getDatabaseNameDependency()); - public static final Parameter EMBEDDING_STORE_POST = new Parameter("s2.embedding.store.post", - "", "端口", "", "number", MODULE_NAME, null, getPostDependency()); + public static final Parameter EMBEDDING_STORE_POST = new Parameter("s2.embedding.store.port", + "", "端口", "", "number", MODULE_NAME, null, getPortDependency()); public static final Parameter EMBEDDING_STORE_USER = new Parameter("s2.embedding.store.user", "", "用户名", "", "string", MODULE_NAME, null, getUserDependency()); @@ -101,10 +101,8 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { private static List getApiKeyDependency() { return getDependency(EMBEDDING_STORE_PROVIDER.getName(), - Lists.newArrayList(EmbeddingStoreType.MILVUS.name(), - EmbeddingStoreType.PGVECTOR.name()), - ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO, - EmbeddingStoreType.PGVECTOR.name(), DEMO)); + Lists.newArrayList(EmbeddingStoreType.MILVUS.name()), + ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO)); } private static List getPathDependency() { @@ -118,7 +116,7 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { Lists.newArrayList(EmbeddingStoreType.MILVUS.name(), EmbeddingStoreType.PGVECTOR.name()), ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "384", - EmbeddingStoreType.PGVECTOR.name(), "768")); + EmbeddingStoreType.PGVECTOR.name(), "512")); } private static List getDatabaseNameDependency() { @@ -129,7 +127,7 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { EmbeddingStoreType.PGVECTOR.name(), "postgres")); } - private static List getPostDependency() { + private static List getPortDependency() { return getDependency(EMBEDDING_STORE_PROVIDER.getName(), Lists.newArrayList(EmbeddingStoreType.PGVECTOR.name()), ImmutableMap.of(EmbeddingStoreType.PGVECTOR.name(), "54333")); @@ -140,12 +138,14 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { Lists.newArrayList(EmbeddingStoreType.MILVUS.name(), EmbeddingStoreType.PGVECTOR.name()), ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus", - EmbeddingStoreType.PGVECTOR.name(), "pgvector")); + EmbeddingStoreType.PGVECTOR.name(), "postgres")); } private static List getPasswordDependency() { return getDependency(EMBEDDING_STORE_PROVIDER.getName(), - Lists.newArrayList(EmbeddingStoreType.MILVUS.name()), - ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus")); + Lists.newArrayList(EmbeddingStoreType.MILVUS.name(), + EmbeddingStoreType.PGVECTOR.name()), + ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus", + EmbeddingStoreType.PGVECTOR.name(), "postgres")); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java index 3ec65c8b2..4b1e81bbd 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java @@ -8,7 +8,6 @@ import lombok.Data; @Builder public class DataItem { - /** * This field uses an underscore (_) at the end. */ private String id; private String bizName; @@ -19,9 +18,10 @@ public class DataItem { private TypeEnums type; - /** * This field uses an underscore (_) at the end. */ private String modelId; + private String domainId; + private String defaultAgg; public String getNewName() { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java index 451db3e01..a801fc0b0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.pojo; +import com.google.common.collect.Lists; import lombok.Data; import java.util.List; @@ -18,5 +19,5 @@ public class ModelRela extends RecordInfo { // left join, inner join, right join, outer join private String joinType; - private List joinConditions; + private List joinConditions = Lists.newArrayList(); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java index a1fc12741..fb00aa5a0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java @@ -1,5 +1,5 @@ package com.tencent.supersonic.common.pojo.enums; public enum TypeEnums { - METRIC, DIMENSION, TAG_OBJECT, TAG, DOMAIN, DATASET, MODEL, UNKNOWN + METRIC, DIMENSION, TAG, DOMAIN, DATASET, MODEL, UNKNOWN } diff --git a/common/src/main/java/dev/langchain4j/pgvector/spring/PgvectorEmbeddingStoreFactory.java b/common/src/main/java/dev/langchain4j/pgvector/spring/PgvectorEmbeddingStoreFactory.java index cde961c8c..b840f6df7 100644 --- a/common/src/main/java/dev/langchain4j/pgvector/spring/PgvectorEmbeddingStoreFactory.java +++ b/common/src/main/java/dev/langchain4j/pgvector/spring/PgvectorEmbeddingStoreFactory.java @@ -26,7 +26,7 @@ public class PgvectorEmbeddingStoreFactory extends BaseEmbeddingStoreFactory { embeddingStore.setPort(storeConfig.getPost()); embeddingStore.setDatabase(storeConfig.getDatabaseName()); embeddingStore.setUser(storeConfig.getUser()); - embeddingStore.setPassword(storeConfig.getApiKey()); + embeddingStore.setPassword(storeConfig.getPassword()); return embeddingStore; } diff --git a/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java b/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java index 2278eefd7..4d8f496d6 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/TextSegmentConvert.java @@ -1,6 +1,7 @@ package dev.langchain4j.store.embedding; import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DataItem; import dev.langchain4j.data.document.Metadata; import dev.langchain4j.data.segment.TextSegment; @@ -17,10 +18,18 @@ public class TextSegmentConvert { public static final String QUERY_ID = "queryId"; public static List convertToEmbedding(List dataItems) { - return dataItems.stream().map(dataItem -> { - Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class); - TextSegment textSegment = TextSegment.from(dataItem.getName(), new Metadata(meta)); - addQueryId(textSegment, dataItem.getId() + dataItem.getType().name().toLowerCase()); + return dataItems.stream().map(item -> { + // suffix with underscore to avoid embedding issue + DataItem newItem = DataItem.builder().domainId(item.getDomainId()) + .bizName(item.getBizName()).type(item.getType()).newName(item.getNewName()) + .defaultAgg(item.getDefaultAgg()).name(item.getName()) + .id(item.getId() + Constants.UNDERLINE) + .modelId(item.getModelId() + Constants.UNDERLINE) + .domainId(item.getDomainId() + Constants.UNDERLINE).build(); + + Map meta = JSONObject.parseObject(JSONObject.toJSONString(newItem), Map.class); + TextSegment textSegment = TextSegment.from(newItem.getName(), new Metadata(meta)); + addQueryId(textSegment, newItem.getId() + newItem.getType().name().toLowerCase()); return textSegment; }).collect(Collectors.toList()); } diff --git a/docker/Dockerfile b/docker/Dockerfile index 666868e91..330c70c32 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,14 +7,13 @@ WORKDIR /usr/src/app # Argument to pass in the supersonic version at build time ARG SUPERSONIC_VERSION -# Install necessary packages, including MySQL client -RUN apt-get update && \ - apt-get install -y default-mysql-client unzip && \ - rm -rf /var/lib/apt/lists/* +RUN apt-get update + +# Install necessary packages, including Postgres client +RUN apt-get update && apt-get install -y postgresql-client # Install the vim editor. -RUN apt-get update && \ - apt-get install -y vim && \ +RUN apt-get update && apt-get install -y vim && \ rm -rf /var/lib/apt/lists/* # Update the package list and install iputils-ping. @@ -40,4 +39,4 @@ WORKDIR /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION} EXPOSE 9080 # Command to run the supersonic daemon RUN chmod +x bin/supersonic-daemon.sh -CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone prd && tail -f /dev/null"] \ No newline at end of file +CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone docker && tail -f /dev/null"] \ No newline at end of file diff --git a/docker/DockerfileS2 b/docker/DockerfileS2 new file mode 100644 index 000000000..d95832640 --- /dev/null +++ b/docker/DockerfileS2 @@ -0,0 +1,32 @@ +# Use an official OpenJDK runtime as a parent image +FROM supersonicbi/supersonic:0.9.8 + +# Set the working directory in the container +WORKDIR /usr/src/app + +# Argument to pass in the supersonic version at build time +ARG SUPERSONIC_VERSION + +# Install necessary packages, including Postgres client +RUN apt-get install -y postgresql-client + +RUN rm /usr/src/app/supersonic-standalone-latest + +# Copy the supersonic standalone zip file into the container +COPY assembly/build/supersonic-standalone-${SUPERSONIC_VERSION}.zip . + +# Unzip the supersonic standalone zip +RUN unzip supersonic-standalone-${SUPERSONIC_VERSION}.zip && \ + rm supersonic-standalone-${SUPERSONIC_VERSION}.zip + +# Create a symbolic link to the supersonic installation directory +RUN ln -s /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION} /usr/src/app/supersonic-standalone-latest + +# Set the working directory to the supersonic installation directory +WORKDIR /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION} + +# Expose the default port +EXPOSE 9080 +# Command to run the supersonic daemon +RUN chmod +x bin/supersonic-daemon.sh +CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone docker && tail -f /dev/null"] \ No newline at end of file diff --git a/docker/docker-compose-run.sh b/docker/docker-compose-run.sh new file mode 100644 index 000000000..7a515ba56 --- /dev/null +++ b/docker/docker-compose-run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +SUPERSONIC_VERSION=0.9.10-SNAPSHOT docker-compose -f docker-compose.yml -p supersonic up \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b6863b202..9b7b0c10a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,51 +1,28 @@ services: - chroma: - image: chromadb/chroma:0.5.3 + postgres: + image: pgvector/pgvector:pg17 privileged: true - container_name: supersonic_chroma - ports: - - "8000:8000" - volumes: - - chroma_data:/chroma - networks: - - supersonic_network - dns: - - 114.114.114.114 - - 8.8.8.8 - - 8.8.4.4 - healthcheck: - test: ["CMD", "curl", "http://0.0.0.0:8000"] - interval: 10s - timeout: 5s - retries: 10 - - mysql: - image: mysql:8.0 - privileged: true - container_name: supersonic_mysql + container_name: supersonic_postgres environment: LANG: 'C.UTF-8' # 设置环境变量 - MYSQL_ROOT_PASSWORD: root_password - MYSQL_DATABASE: supersonic_db - MYSQL_USER: supersonic_user - MYSQL_PASSWORD: supersonic_password + POSTGRES_ROOT_PASSWORD: root_password + POSTGRES_DATABASE: postgres + POSTGRES_USER: supersonic_user + POSTGRES_PASSWORD: supersonic_password ports: - - "13306:3306" + - "15432:5432" volumes: - - mysql_data:/var/lib/mysql + - postgres_data:/var/lib/postgresql networks: - supersonic_network dns: - 114.114.114.114 - 8.8.8.8 - 8.8.4.4 - depends_on: - chroma: - condition: service_healthy healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] - interval: 10s - timeout: 5s + test: ["CMD-SHELL", "sh -c 'pg_isready -U supersonic_user -d postgres'"] + interval: 30s + timeout: 10s retries: 5 db_init: @@ -53,22 +30,21 @@ services: privileged: true container_name: supersonic_db_init depends_on: - mysql: + postgres: condition: service_healthy networks: - supersonic_network command: > sh -c " - sleep 15 && - if ! mysql -h supersonic_mysql -usupersonic_user -psupersonic_password -e 'use supersonic_db; show tables;' | grep -q 's2_database'; then - mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql.sql && - mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql-demo.sql && - mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql.sql && - mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql-demo.sql - else - echo 'Database already initialized.' - fi - " + if ! PGPASSWORD=supersonic_password psql -h supersonic_postgres -U supersonic_user -d postgres -c 'select * from s2_database limit 1' > /dev/null; + then + PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-postgres.sql + PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-postgres-demo.sql + PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-postgres.sql + PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-postgres-demo.sql + else + echo 'Database already initialized.' + fi" dns: - 114.114.114.114 - 8.8.8.8 @@ -79,17 +55,14 @@ services: privileged: true container_name: supersonic_standalone environment: - DB_HOST: supersonic_mysql - DB_NAME: supersonic_db + DB_HOST: supersonic_postgres + DB_NAME: postgres DB_USERNAME: supersonic_user DB_PASSWORD: supersonic_password - CHROMA_HOST: supersonic_chroma ports: - "9080:9080" depends_on: - chroma: - condition: service_healthy - mysql: + postgres: condition: service_healthy db_init: condition: service_completed_successfully @@ -112,8 +85,7 @@ services: # propagation: rprivate # create_host_path: true volumes: - mysql_data: - chroma_data: + postgres_data: supersonic_data: networks: diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java index 79670baf0..51f6112c5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java @@ -15,7 +15,7 @@ public class ColumnSchema { private FieldType filedType; - private AggOperatorEnum agg; + private AggOperatorEnum agg = AggOperatorEnum.SUM; private String name; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java similarity index 77% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java index 594e9915b..6ed5b0450 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dim.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java @@ -9,7 +9,7 @@ import lombok.NoArgsConstructor; @Data @AllArgsConstructor @NoArgsConstructor -public class Dim { +public class Dimension { private String name; @@ -27,16 +27,14 @@ public class Dim { private String description; - private int isTag; - - public Dim(String name, String bizName, DimensionType type, Integer isCreateDimension) { + public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension) { this.name = name; this.type = type; this.isCreateDimension = isCreateDimension; this.bizName = bizName; } - public Dim(String name, String bizName, DimensionType type, Integer isCreateDimension, + public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension, String expr, String dateFormat, DimensionTimeTypeParams typeParams) { this.name = name; this.type = type; @@ -47,8 +45,8 @@ public class Dim { this.bizName = bizName; } - public static Dim getDefault() { - return new Dim("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date", + public static Dimension getDefault() { + return new Dimension("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date", Constants.DAY_FORMAT, new DimensionTimeTypeParams("false", "day")); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index cc1424e03..8c9fc1753 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -24,7 +24,7 @@ public class ModelDetail { private List identifiers = Lists.newArrayList(); - private List dimensions = Lists.newArrayList(); + private List dimensions = Lists.newArrayList(); private List measures = Lists.newArrayList(); @@ -39,7 +39,7 @@ public class ModelDetail { return sqlQuery; } - public List filterTimeDims() { + public List filterTimeDims() { if (CollectionUtils.isEmpty(dimensions)) { return Lists.newArrayList(); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/FieldType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/FieldType.java index ce24f76c9..0f0e8afa3 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/FieldType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/FieldType.java @@ -1,5 +1,5 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum FieldType { - primary_key, foreign_key, partition_time, time, dimension, measure; + primary_key, foreign_key, partition_time, time, categorical, measure; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java index 5e62fcf68..518522696 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java @@ -32,8 +32,6 @@ public class DimensionReq extends SchemaItem { private DataTypeEnums dataType; - private int isTag; - private Map ext; private DimensionTimeTypeParams typeParams; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java index 9385bb28b..8dc9ffa0a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java @@ -9,6 +9,10 @@ import java.util.List; @Data public class ModelBuildReq { + private String name; + + private String bizName; + private Long databaseId; private Long domainId; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 752f193c6..2880111f2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -211,7 +211,9 @@ public class QueryStructReq extends SemanticQueryReq { SelectItem selectExpressionItem = new SelectItem(function); String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName; - selectExpressionItem.setAlias(new Alias(alias)); + if (!alias.equals(columnName)) { + selectExpressionItem.setAlias(new Alias(alias)); + } return selectExpressionItem; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java index b5c70830d..eaf1a8937 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java @@ -18,6 +18,8 @@ public class DimensionResp extends SchemaItem { private Long modelId; + private Long domainId; + private DimensionType type; private String expr; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java index d4efdcb29..afead8d78 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.Identify; @@ -62,7 +62,7 @@ public class ModelResp extends SchemaItem { return isOpen != null && isOpen == 1; } - public List getTimeDimension() { + public List getTimeDimension() { if (modelDetail == null) { return Lists.newArrayList(); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java index 97bef6f69..698ebafcf 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java @@ -19,6 +19,7 @@ public class DictWord { private String word; private String nature; private String natureWithFrequency; + private String alias; @Override public boolean equals(Object o) { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java index a6dabb953..7cf018360 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java @@ -1,11 +1,14 @@ package com.tencent.supersonic.headless.chat.knowledge; +import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -14,6 +17,31 @@ import java.util.stream.Collectors; @Service @Slf4j public class KnowledgeBaseService { + private static volatile Map> dimValueAliasMap = new HashMap<>(); + + public static Map> getDimValueAlias() { + return dimValueAliasMap; + } + + public static List addDimValueAlias(Long dimId, List newWords) { + List dimValueAlias = + dimValueAliasMap.containsKey(dimId) ? dimValueAliasMap.get(dimId) + : new ArrayList<>(); + Set wordSet = + dimValueAlias + .stream().map(word -> String.format("%s_%s_%s", + word.getNatureWithFrequency(), word.getWord(), word.getAlias())) + .collect(Collectors.toSet()); + for (DictWord dictWord : newWords) { + String key = String.format("%s_%s_%s", dictWord.getNatureWithFrequency(), + dictWord.getWord(), dictWord.getAlias()); + if (!wordSet.contains(key)) { + dimValueAlias.add(dictWord); + } + } + dimValueAliasMap.put(dimId, dimValueAlias); + return dimValueAlias; + } public void updateSemanticKnowledge(List natures) { @@ -41,6 +69,11 @@ public class KnowledgeBaseService { } // 2. update online knowledge + if (CollectionUtils.isNotEmpty(dimValueAliasMap)) { + for (Long dimId : dimValueAliasMap.keySet()) { + natures.addAll(dimValueAliasMap.get(dimId)); + } + } updateOnlineKnowledge(natures); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java index c7511326f..79cefa92f 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java @@ -12,6 +12,8 @@ import com.hankcs.hanlp.dictionary.other.CharTable; import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.utility.LexiconUtility; import com.hankcs.hanlp.utility.TextUtility; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; import java.io.BufferedOutputStream; @@ -103,7 +105,22 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { String word = getWordBySpace(param[0]); if (isLetters) { original = word; - word = word.toLowerCase(); + // word = word.toLowerCase(); + // 加入小写别名 + if (!original.equals(word.toLowerCase())) { + DictWord dictWord = new DictWord(); + String nature = param[1]; + dictWord.setNatureWithFrequency( + String.format("%s " + Constants.DEFAULT_FREQUENCY, nature)); + dictWord.setWord(word); + dictWord.setAlias(word.toLowerCase()); + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length >= 2) { + Long dimId = Long.parseLong( + nature.split(DictWordType.NATURE_SPILT)[split.length - 1]); + KnowledgeBaseService.addDimValueAlias(dimId, Arrays.asList(dictWord)); + } + } } if (natureCount == 0) { attribute = new CoreDictionary.Attribute(defaultNature); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java index cfce186d8..33450eda1 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java @@ -8,12 +8,15 @@ import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.knowledge.DatabaseMapResult; +import com.tencent.supersonic.headless.chat.knowledge.DictWord; import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService; import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; import com.tencent.supersonic.headless.chat.utils.EditDistanceUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; import java.util.HashSet; @@ -83,12 +86,32 @@ public class KeywordMapper extends BaseMapper { .element(element).frequency(frequency).word(hanlpMapResult.getName()) .similarity(hanlpMapResult.getSimilarity()) .detectWord(hanlpMapResult.getDetectWord()).build(); - + // doDimValueAliasLogic 将维度值别名进行替换成真实维度值 + doDimValueAliasLogic(schemaElementMatch); addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch); } } } + private void doDimValueAliasLogic(SchemaElementMatch schemaElementMatch) { + SchemaElement element = schemaElementMatch.getElement(); + if (SchemaElementType.VALUE.equals(element.getType())) { + Long dimId = element.getId(); + String word = schemaElementMatch.getWord(); + Map> dimValueAlias = KnowledgeBaseService.getDimValueAlias(); + if (Objects.nonNull(dimId) && StringUtils.isNotEmpty(word) + && dimValueAlias.containsKey(dimId)) { + Map aliasAndDictMap = dimValueAlias.get(dimId).stream() + .collect(Collectors.toMap(dictWord -> dictWord.getAlias(), + dictWord -> dictWord, (v1, v2) -> v2)); + if (aliasAndDictMap.containsKey(word)) { + String wordTech = aliasAndDictMap.get(word).getWord(); + schemaElementMatch.setWord(wordTech); + } + } + } + } + private void convertMapResultToMapInfo(ChatQueryContext chatQueryContext, List mapResults) { for (DatabaseMapResult match : mapResults) { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TimeFieldMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TimeFieldMapper.java new file mode 100644 index 000000000..667b405e0 --- /dev/null +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TimeFieldMapper.java @@ -0,0 +1,37 @@ +package com.tencent.supersonic.headless.chat.mapper; + +import com.tencent.supersonic.common.pojo.enums.Text2SQLType; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.chat.ChatQueryContext; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class TimeFieldMapper extends BaseMapper { + + @Override + public void doMap(ChatQueryContext chatQueryContext) { + if (chatQueryContext.getRequest().getText2SQLType().equals(Text2SQLType.ONLY_RULE)) { + return; + } + + Map schemaMap = + chatQueryContext.getSemanticSchema().getDataSetSchemaMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + List timeDims = entry.getValue().getDimensions().stream() + .filter(dim -> dim.getTimeFormat() != null).collect(Collectors.toList()); + for (SchemaElement schemaElement : timeDims) { + chatQueryContext.getMapInfo().getMatchedElements(entry.getKey()) + .add(SchemaElementMatch.builder().word(schemaElement.getName()) + .element(schemaElement).detectWord(schemaElement.getName()) + .similarity(1.0).build()); + } + } + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java index b03521996..a61cc7d76 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java @@ -84,7 +84,7 @@ public abstract class BaseDbAdaptor implements DbAdaptor { return connection.getMetaData(); } - protected static FieldType classifyColumnType(String typeName) { + public FieldType classifyColumnType(String typeName) { switch (typeName.toUpperCase()) { case "INT": case "INTEGER": @@ -101,7 +101,7 @@ public abstract class BaseDbAdaptor implements DbAdaptor { case "TIMESTAMP": return FieldType.time; default: - return FieldType.dimension; + return FieldType.categorical; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java index f2e650177..94862f633 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.api.pojo.enums.FieldType; import com.tencent.supersonic.headless.core.pojo.ConnectInfo; import java.sql.SQLException; @@ -19,4 +20,6 @@ public interface DbAdaptor { List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException; + + FieldType classifyColumnType(String typeName); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java index ff4cefe5a..f04fb0e5f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java @@ -114,7 +114,8 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { return dbColumns; } - protected static FieldType classifyColumnType(String typeName) { + @Override + public FieldType classifyColumnType(String typeName) { switch (typeName.toUpperCase()) { case "INT": case "INTEGER": @@ -141,7 +142,7 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { case "CHARACTER": case "UUID": default: - return FieldType.dimension; + return FieldType.categorical; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java index 47ff61e2a..cee62013d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java @@ -61,13 +61,18 @@ public class SqlQueryConverter implements QueryConverter { List metricSchemas = getMetrics(semanticSchemaResp, allFields); List metrics = metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList()); - AggOption aggOption = getAggOption(sqlQueryParam.getSql(), metricSchemas); Set dimensions = getDimensions(semanticSchemaResp, allFields); OntologyQueryParam ontologyQueryParam = new OntologyQueryParam(); ontologyQueryParam.getMetrics().addAll(metrics); ontologyQueryParam.getDimensions().addAll(dimensions); - ontologyQueryParam.setAggOption(aggOption); - ontologyQueryParam.setNativeQuery(!AggOption.isAgg(aggOption)); + AggOption sqlQueryAggOption = getAggOption(sqlQueryParam.getSql(), metricSchemas); + // if sql query itself has aggregation, ontology query just returns detail + if (sqlQueryAggOption.equals(AggOption.AGGREGATION)) { + ontologyQueryParam.setAggOption(AggOption.NATIVE); + } else if (sqlQueryAggOption.equals(AggOption.NATIVE) && !metrics.isEmpty()) { + ontologyQueryParam.setAggOption(AggOption.DEFAULT); + } + ontologyQueryParam.setNativeQuery(!AggOption.isAgg(ontologyQueryParam.getAggOption())); queryStatement.setOntologyQueryParam(ontologyQueryParam); generateDerivedMetric(sqlGenerateUtils, queryStatement); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java index ef89dce5c..c0c3d79e6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java @@ -57,7 +57,11 @@ public class StructQueryConverter implements QueryConverter { .map(Aggregator::getColumn).collect(Collectors.toList())); String where = sqlGenerateUtils.generateWhere(structQueryParam, null); ontologyQueryParam.setWhere(where); - ontologyQueryParam.setAggOption(AggOption.AGGREGATION); + if (ontologyQueryParam.getMetrics().isEmpty()) { + ontologyQueryParam.setAggOption(AggOption.NATIVE); + } else { + ontologyQueryParam.setAggOption(AggOption.DEFAULT); + } ontologyQueryParam.setNativeQuery(structQueryParam.getQueryType().isNativeAggQuery()); ontologyQueryParam.setOrder(structQueryParam.getOrders().stream() .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java index 66113ab9b..75d32ad9f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java @@ -26,8 +26,8 @@ public class DataModelNode extends SemanticNode { sqlTable = dataModel.getSqlQuery(); } else if (dataModel.getTableQuery() != null && !dataModel.getTableQuery().isEmpty()) { if (dataModel.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { - String fullTableName = Arrays.stream(dataModel.getTableQuery().split("\\.")) - .collect(Collectors.joining(".public.")); + String fullTableName = + String.join(".public.", dataModel.getTableQuery().split("\\.")); sqlTable = "select * from " + fullTableName; } else { sqlTable = "select * from " + dataModel.getTableQuery(); @@ -64,7 +64,7 @@ public class DataModelNode extends SemanticNode { for (Dimension d : datasource.getDimensions()) { List identifiers = expand(SemanticNode.parse(d.getExpr(), scope, engineType), scope); - identifiers.stream().forEach(i -> dimensions.add(i.toString())); + identifiers.forEach(i -> dimensions.add(i.toString())); dimensions.add(d.getName()); } for (Identify i : datasource.getIdentifiers()) { @@ -73,7 +73,7 @@ public class DataModelNode extends SemanticNode { for (Measure m : datasource.getMeasures()) { List identifiers = expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope); - identifiers.stream().forEach(i -> { + identifiers.forEach(i -> { if (!dimensions.contains(i.toString())) { metrics.add(i.toString()); } @@ -127,7 +127,7 @@ public class DataModelNode extends SemanticNode { } public static String getNames(List dataModelList) { - return dataModelList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); + return dataModelList.stream().map(DataModel::getName).collect(Collectors.joining("_")); } public static void getQueryDimensionMeasure(Ontology ontology, OntologyQueryParam queryParam, @@ -138,12 +138,12 @@ public class DataModelNode extends SemanticNode { : d) .collect(Collectors.toSet())); Set schemaMetricName = - ontology.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet()); + ontology.getMetrics().stream().map(Metric::getName).collect(Collectors.toSet()); ontology.getMetrics().stream().filter(m -> queryParam.getMetrics().contains(m.getName())) - .forEach(m -> m.getMetricTypeParams().getMeasures().stream() + .forEach(m -> m.getMetricTypeParams().getMeasures() .forEach(mm -> queryMeasures.add(mm.getName()))); queryParam.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)) - .forEach(m -> queryMeasures.add(m)); + .forEach(queryMeasures::add); } public static void mergeQueryFilterDimensionMeasure(Ontology ontology, @@ -155,13 +155,13 @@ public class DataModelNode extends SemanticNode { FilterNode.getFilterField(parse(queryParam.getWhere(), scope, engineType), filterConditions); Set queryMeasures = new HashSet<>(measures); - Set schemaMetricName = ontology.getMetrics().stream().map(m -> m.getName()) - .collect(Collectors.toSet()); + Set schemaMetricName = + ontology.getMetrics().stream().map(Metric::getName).collect(Collectors.toSet()); for (String filterCondition : filterConditions) { if (schemaMetricName.contains(filterCondition)) { ontology.getMetrics().stream() .filter(m -> m.getName().equalsIgnoreCase(filterCondition)) - .forEach(m -> m.getMetricTypeParams().getMeasures().stream() + .forEach(m -> m.getMetricTypeParams().getMeasures() .forEach(mm -> queryMeasures.add(mm.getName()))); continue; } @@ -196,8 +196,8 @@ public class DataModelNode extends SemanticNode { } // second, traverse the ontology to find other related dataModels - List relatedDataModels = findRelatedModelsByRelation(ontology, baseDataModel, - queryDimensions, queryMeasures); + List relatedDataModels = findRelatedModelsByRelation(ontology, queryParam, + baseDataModel, queryDimensions, queryMeasures); if (CollectionUtils.isEmpty(relatedDataModels)) { relatedDataModels = findRelatedModelsByIdentifier(ontology, baseDataModel, queryDimensions, queryMeasures); @@ -255,7 +255,7 @@ public class DataModelNode extends SemanticNode { .collect(Collectors.toSet()); Set baseDimensions = baseDataModel.getDimensions().stream().map(Dimension::getName) .collect(Collectors.toSet()); - baseDataModel.getIdentifiers().stream().forEach(i -> baseDimensions.add(i.getName())); + baseDataModel.getIdentifiers().forEach(i -> baseDimensions.add(i.getName())); baseMeasures.retainAll(queryMeasures); if (baseMeasures.size() < queryMeasures.size()) { @@ -282,7 +282,8 @@ public class DataModelNode extends SemanticNode { } private static List findRelatedModelsByRelation(Ontology ontology, - DataModel baseDataModel, Set queryDimensions, Set queryMeasures) { + OntologyQueryParam queryParam, DataModel baseDataModel, Set queryDimensions, + Set queryMeasures) { Set joinDataModelNames = new HashSet<>(); List joinDataModels = new ArrayList<>(); Set before = new HashSet<>(); @@ -295,7 +296,7 @@ public class DataModelNode extends SemanticNode { visitJoinRelations, sortedJoinRelation); ontology.getJoinRelations().stream() .filter(j -> !visitJoinRelations.contains(j.getId())) - .forEach(j -> sortedJoinRelation.add(j)); + .forEach(sortedJoinRelation::add); for (JoinRelation joinRelation : sortedJoinRelation) { if (!before.contains(joinRelation.getLeft()) && !before.contains(joinRelation.getRight())) { @@ -305,13 +306,17 @@ public class DataModelNode extends SemanticNode { boolean isRight = before.contains(joinRelation.getLeft()); DataModel other = isRight ? ontology.getDataModelMap().get(joinRelation.getRight()) : ontology.getDataModelMap().get(joinRelation.getLeft()); + String joinDimName = isRight ? joinRelation.getJoinCondition().get(0).getRight() + : joinRelation.getJoinCondition().get(0).getLeft(); if (!queryDimensions.isEmpty()) { Set linkDimension = other.getDimensions().stream() - .map(dd -> dd.getName()).collect(Collectors.toSet()); - other.getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName())); + .map(Dimension::getName).collect(Collectors.toSet()); + other.getIdentifiers().forEach(i -> linkDimension.add(i.getName())); linkDimension.retainAll(queryDimensions); if (!linkDimension.isEmpty()) { isMatch = true; + // joinDim should be added to the query dimension + queryParam.getDimensions().add(joinDimName); } } Set linkMeasure = other.getMeasures().stream().map(Measure::getName) @@ -322,7 +327,7 @@ public class DataModelNode extends SemanticNode { } if (!isMatch && ontology.getDimensionMap().containsKey(other.getName())) { Set linkDimension = ontology.getDimensionMap().get(other.getName()) - .stream().map(dd -> dd.getName()).collect(Collectors.toSet()); + .stream().map(Dimension::getName).collect(Collectors.toSet()); linkDimension.retainAll(queryDimensions); if (!linkDimension.isEmpty()) { isMatch = true; @@ -382,15 +387,14 @@ public class DataModelNode extends SemanticNode { if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) { continue; } - Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName()) - .filter(i -> baseIdentifiers.contains(i)).count(); + long identifierNum = entry.getValue().getIdentifiers().stream().map(Identify::getName) + .filter(baseIdentifiers::contains).count(); if (identifierNum > 0) { boolean isMatch = false; if (!queryDimension.isEmpty()) { Set linkDimension = entry.getValue().getDimensions().stream() - .map(dd -> dd.getName()).collect(Collectors.toSet()); - entry.getValue().getIdentifiers().stream() - .forEach(i -> linkDimension.add(i.getName())); + .map(Dimension::getName).collect(Collectors.toSet()); + entry.getValue().getIdentifiers().forEach(i -> linkDimension.add(i.getName())); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { isMatch = true; @@ -398,7 +402,7 @@ public class DataModelNode extends SemanticNode { } if (!measures.isEmpty()) { Set linkMeasure = entry.getValue().getMeasures().stream() - .map(mm -> mm.getName()).collect(Collectors.toSet()); + .map(Measure::getName).collect(Collectors.toSet()); linkMeasure.retainAll(measures); if (!linkMeasure.isEmpty()) { isMatch = true; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java index 10aea58c5..48b9a47ab 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java @@ -15,6 +15,6 @@ public class OntologyQueryParam { private String where; private Long limit; private List order; - private boolean nativeQuery = false; - private AggOption aggOption = AggOption.DEFAULT; + private boolean nativeQuery = true; + private AggOption aggOption = AggOption.NATIVE; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java index 2c966477c..d17b301e8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; @@ -20,11 +20,11 @@ public class SysTimeDimensionBuilder { Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b", Pattern.CASE_INSENSITIVE); - public static void addSysTimeDimension(List dims, DbAdaptor engineAdaptor) { + public static void addSysTimeDimension(List dims, DbAdaptor engineAdaptor) { log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor); - Dim timeDim = getTimeDim(dims); + Dimension timeDim = getTimeDim(dims); if (timeDim == null) { - timeDim = Dim.getDefault(); + timeDim = Dimension.getDefault(); // todo not find the time dimension return; } @@ -34,8 +34,8 @@ public class SysTimeDimensionBuilder { log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor); } - private static Dim generateSysDayDimension(Dim timeDim, DbAdaptor engineAdaptor) { - Dim dim = new Dim(); + private static Dimension generateSysDayDimension(Dimension timeDim, DbAdaptor engineAdaptor) { + Dimension dim = new Dimension(); dim.setBizName(TimeDimensionEnum.DAY.getName()); dim.setType(DimensionType.partition_time); dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(), @@ -47,8 +47,8 @@ public class SysTimeDimensionBuilder { return dim; } - private static Dim generateSysWeekDimension(Dim timeDim, DbAdaptor engineAdaptor) { - Dim dim = new Dim(); + private static Dimension generateSysWeekDimension(Dimension timeDim, DbAdaptor engineAdaptor) { + Dimension dim = new Dimension(); dim.setBizName(TimeDimensionEnum.WEEK.getName()); dim.setType(DimensionType.partition_time); dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(), @@ -60,8 +60,8 @@ public class SysTimeDimensionBuilder { return dim; } - private static Dim generateSysMonthDimension(Dim timeDim, DbAdaptor engineAdaptor) { - Dim dim = new Dim(); + private static Dimension generateSysMonthDimension(Dimension timeDim, DbAdaptor engineAdaptor) { + Dimension dim = new Dimension(); dim.setBizName(TimeDimensionEnum.MONTH.getName()); dim.setType(DimensionType.partition_time); dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(), @@ -79,7 +79,8 @@ public class SysTimeDimensionBuilder { } // Check whether the time field contains keywords,Generation time expression - private static String generateTimeExpr(Dim timeDim, String dateType, DbAdaptor engineAdaptor) { + private static String generateTimeExpr(Dimension timeDim, String dateType, + DbAdaptor engineAdaptor) { String bizName = timeDim.getBizName(); String dateFormat = timeDim.getDateFormat(); if (containsTimeKeyword(bizName)) { @@ -90,8 +91,8 @@ public class SysTimeDimensionBuilder { } } - private static Dim getTimeDim(List timeDims) { - for (Dim dim : timeDims) { + private static Dimension getTimeDim(List timeDims) { + for (Dimension dim : timeDims) { if (dim.getType().equals(DimensionType.partition_time)) { return dim; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index 306ac693a..22d42c9c9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -207,7 +207,7 @@ public class S2SemanticLayerService implements SemanticLayerService { ModelResp modelResp = modelResps.get(0); String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(), modelResp.getName()); - List timeDims = modelResp.getTimeDimension(); + List timeDims = modelResp.getTimeDimension(); if (CollectionUtils.isNotEmpty(timeDims)) { sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(), diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java index 6561821e0..dc56f1700 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java @@ -27,7 +27,7 @@ public class SchemaDictUpdateListener implements ApplicationListener dictWord.setWord(dataItem.getName()); String sign = DictWordType.NATURE_SPILT; String suffixNature = DictWordType.getSuffixNature(dataItem.getType()); - String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature; + String nature = sign + dataItem.getModelId() + sign + dataItem.getId() + suffixNature; String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; dictWord.setNature(nature); dictWord.setNatureWithFrequency(natureWithFrequency); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 44b2fb6bd..8075d421a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.server.manager; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.ModelDetail; @@ -18,7 +18,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; import java.util.Objects; import java.util.stream.Collectors; @@ -54,7 +53,7 @@ public class ModelYamlManager { return dataModelYamlTpl; } - public static DimensionYamlTpl convert(Dim dim) { + public static DimensionYamlTpl convert(Dimension dim) { DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl(); BeanUtils.copyProperties(dim, dimensionYamlTpl); dimensionYamlTpl.setName(dim.getBizName()); @@ -85,15 +84,4 @@ public class ModelYamlManager { return identifyYamlTpl; } - private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) { - Measure measure = new Measure(); - measure.setExpr("1"); - if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) { - measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName()); - } - measure.setAgg("count"); - measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt")); - measure.setIsCreateMetric(1); - datasourceDetail.getMeasures().add(measure); - } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java index b27c25169..324c0cb61 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java @@ -23,7 +23,7 @@ public class RuleSemanticModeller implements SemanticModeller { private ColumnSchema convert(DBColumn dbColumn) { ColumnSchema columnSchema = new ColumnSchema(); - columnSchema.setName(dbColumn.getComment()); + columnSchema.setName(dbColumn.getColumnName()); columnSchema.setColumnName(dbColumn.getColumnName()); columnSchema.setComment(dbColumn.getComment()); columnSchema.setDataType(dbColumn.getDataType()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java index 096fccf69..eb1ad3716 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java @@ -16,6 +16,7 @@ public class ModelDO { private Long domainId; + @Deprecated private Long tagObjectId; private String name; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java index 67a7edabf..4235390d7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java @@ -28,6 +28,7 @@ public class QueryStatDO { private String queryStructCmd; @TableField("struct_cmd_md5") private String queryStructCmdMd5; + @TableField("\"sql\"") private String sql; private String sqlMd5; private String queryEngine; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java index 746afe6a5..741220eec 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; -import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import org.apache.ibatis.annotations.Mapper; @@ -12,11 +11,7 @@ public interface DimensionDOCustomMapper { void batchInsert(List dimensionDOS); - void batchUpdate(List dimensionDOS); - void batchUpdateStatus(List dimensionDOS); - List query(DimensionFilter dimensionFilter); - List queryDimensions(DimensionsFilter dimensionsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java index 651e927d3..f53795790 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; -import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import org.apache.ibatis.annotations.Mapper; @@ -20,7 +19,5 @@ public interface MetricDOCustomMapper { void updateClassificationsBatch(List metricDOS); - List query(MetricFilter metricFilter); - List queryMetrics(MetricsFilter metricsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java index 32095d40f..60e0b8743 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java @@ -1,15 +1,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.tencent.supersonic.headless.api.pojo.QueryStat; -import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.server.persistence.dataobject.QueryStatDO; import org.apache.ibatis.annotations.Mapper; -import java.util.List; - @Mapper public interface StatMapper extends BaseMapper { - - List getStatInfo(ItemUseReq itemUseCommend); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java index 55ffe1d11..77ea46813 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java @@ -118,6 +118,7 @@ public class DictRepositoryImpl implements DictRepository { wrapper.lambda().and(qw -> qw.like(DictTaskDO::getName, key).or() .like(DictTaskDO::getDescription, key).or().like(DictTaskDO::getConfig, key)); } + wrapper.lambda().orderByDesc(DictTaskDO::getCreatedAt); return dictTaskMapper.selectList(wrapper); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index 4fe25fcda..95c48d6f4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -1,14 +1,17 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCustomMapper; import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Objects; @Service public class DimensionRepositoryImpl implements DimensionRepository { @@ -50,7 +53,43 @@ public class DimensionRepositoryImpl implements DimensionRepository { @Override public List getDimension(DimensionFilter dimensionFilter) { - return dimensionDOCustomMapper.query(dimensionFilter); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.lambda().ne(DimensionDO::getStatus, 3); + if (Objects.nonNull(dimensionFilter.getIds()) && !dimensionFilter.getIds().isEmpty()) { + queryWrapper.lambda().in(DimensionDO::getId, dimensionFilter.getIds()); + } + if (StringUtils.isNotBlank(dimensionFilter.getId())) { + queryWrapper.lambda().eq(DimensionDO::getId, dimensionFilter.getId()); + } + if (Objects.nonNull(dimensionFilter.getModelIds()) + && !dimensionFilter.getModelIds().isEmpty()) { + queryWrapper.lambda().in(DimensionDO::getModelId, dimensionFilter.getModelIds()); + } + if (StringUtils.isNotBlank(dimensionFilter.getName())) { + queryWrapper.lambda().like(DimensionDO::getName, dimensionFilter.getName()); + } + if (StringUtils.isNotBlank(dimensionFilter.getId())) { + queryWrapper.lambda().like(DimensionDO::getBizName, dimensionFilter.getBizName()); + } + if (Objects.nonNull(dimensionFilter.getStatus())) { + queryWrapper.lambda().eq(DimensionDO::getStatus, dimensionFilter.getStatus()); + } + if (Objects.nonNull(dimensionFilter.getSensitiveLevel())) { + queryWrapper.lambda().eq(DimensionDO::getSensitiveLevel, + dimensionFilter.getSensitiveLevel()); + } + if (StringUtils.isNotBlank(dimensionFilter.getCreatedBy())) { + queryWrapper.lambda().eq(DimensionDO::getCreatedBy, dimensionFilter.getCreatedBy()); + } + if (StringUtils.isNotBlank(dimensionFilter.getKey())) { + String key = dimensionFilter.getKey(); + queryWrapper.lambda().like(DimensionDO::getName, key).or() + .like(DimensionDO::getBizName, key).or().like(DimensionDO::getDescription, key) + .or().like(DimensionDO::getAlias, key).or() + .like(DimensionDO::getCreatedBy, key); + } + + return dimensionDOMapper.selectList(queryWrapper); } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java index f79d8eabd..3cf5eb0b9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java @@ -9,9 +9,11 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefa import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import java.util.List; +import java.util.Objects; @Component public class MetricRepositoryImpl implements MetricRepository { @@ -73,7 +75,46 @@ public class MetricRepositoryImpl implements MetricRepository { @Override public List getMetric(MetricFilter metricFilter) { - return metricDOCustomMapper.query(metricFilter); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.lambda().ne(MetricDO::getStatus, 3); + if (Objects.nonNull(metricFilter.getIds()) && !metricFilter.getIds().isEmpty()) { + queryWrapper.lambda().in(MetricDO::getId, metricFilter.getIds()); + } + if (StringUtils.isNotBlank(metricFilter.getId())) { + queryWrapper.lambda().eq(MetricDO::getId, metricFilter.getId()); + } + if (Objects.nonNull(metricFilter.getModelIds()) && !metricFilter.getModelIds().isEmpty()) { + queryWrapper.lambda().in(MetricDO::getModelId, metricFilter.getModelIds()); + } + if (StringUtils.isNotBlank(metricFilter.getType())) { + queryWrapper.lambda().eq(MetricDO::getType, metricFilter.getType()); + } + if (StringUtils.isNotBlank(metricFilter.getName())) { + queryWrapper.lambda().like(MetricDO::getName, metricFilter.getName()); + } + if (StringUtils.isNotBlank(metricFilter.getId())) { + queryWrapper.lambda().like(MetricDO::getBizName, metricFilter.getBizName()); + } + if (Objects.nonNull(metricFilter.getStatus())) { + queryWrapper.lambda().eq(MetricDO::getStatus, metricFilter.getStatus()); + } + if (Objects.nonNull(metricFilter.getSensitiveLevel())) { + queryWrapper.lambda().eq(MetricDO::getSensitiveLevel, metricFilter.getSensitiveLevel()); + } + if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) { + queryWrapper.lambda().eq(MetricDO::getCreatedBy, metricFilter.getCreatedBy()); + } + if (Objects.nonNull(metricFilter.getIsPublish()) && metricFilter.getIsPublish() == 1) { + queryWrapper.lambda().eq(MetricDO::getIsPublish, metricFilter.getIsPublish()); + } + if (StringUtils.isNotBlank(metricFilter.getKey())) { + String key = metricFilter.getKey(); + queryWrapper.lambda().like(MetricDO::getName, key).or().like(MetricDO::getBizName, key) + .or().like(MetricDO::getDescription, key).or().like(MetricDO::getAlias, key) + .or().like(MetricDO::getCreatedBy, key); + } + + return metricDOMapper.selectList(queryWrapper); } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java index 62bd35806..8ca7461b3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java @@ -51,7 +51,7 @@ public class QueryRuleRepositoryImpl implements QueryRuleRepository { QueryWrapper wrapperSys = new QueryWrapper<>(); // 返回系统设置的规则 - wrapperSys.or().eq("priority", 0L); + wrapperSys.lambda().or().eq(QueryRuleDO::getPriority, 0L); List queryRuleDOListSys = mapper.selectList(wrapperSys); queryRuleDOList.addAll(queryRuleDOListSys); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java index f670242c0..064f38b78 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.tencent.supersonic.common.pojo.enums.TypeEnums; @@ -47,7 +48,7 @@ public class StatRepositoryImpl implements StatRepository { @SneakyThrows public List getStatInfo(ItemUseReq itemUseReq) { List result = new ArrayList<>(); - List statInfos = statMapper.getStatInfo(itemUseReq); + List statInfos = getQueryStats(itemUseReq); Map map = new ConcurrentHashMap<>(); statInfos.stream().forEach(stat -> { String dimensions = stat.getDimensions(); @@ -70,6 +71,21 @@ public class StatRepositoryImpl implements StatRepository { .collect(Collectors.toList()); } + private List getQueryStats(ItemUseReq itemUseReq) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (Objects.nonNull(itemUseReq.getModelId())) { + queryWrapper.lambda().eq(QueryStatDO::getModelId, itemUseReq.getModelId()); + } + if (Objects.nonNull(itemUseReq.getModelIds()) && !itemUseReq.getModelIds().isEmpty()) { + queryWrapper.lambda().in(QueryStatDO::getModelId, itemUseReq.getModelIds()); + } + if (Objects.nonNull(itemUseReq.getMetric())) { + queryWrapper.lambda().like(QueryStatDO::getMetrics, itemUseReq.getMetric()); + } + + return statMapper.selectList(queryWrapper); + } + private void updateStatMapInfo(Map map, String dimensions, String type, Long dataSetId) { if (StringUtils.isNotEmpty(dimensions)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java index b79413b64..864757002 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java @@ -47,7 +47,7 @@ public interface DimensionService { void sendDimensionEventBatch(List modelIds, EventType eventType); - DataEvent getDataEvent(); + DataEvent getAllDataEvents(); Boolean updateDimValueAlias(DimValueAliasReq req, User user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java index 8fe99042c..8abff2ba5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java @@ -79,7 +79,7 @@ public class DataSetServiceImpl extends ServiceImpl DataSetDO dataSetDO = convert(dataSetReq); dataSetDO.setStatus(StatusEnum.ONLINE.getCode()); DataSetResp dataSetResp = convert(dataSetDO); - conflictCheck(dataSetResp); + // conflictCheck(dataSetResp); save(dataSetDO); dataSetResp.setId(dataSetDO.getId()); return dataSetResp; @@ -90,7 +90,7 @@ public class DataSetServiceImpl extends ServiceImpl dataSetReq.updatedBy(user.getName()); DataSetDO dataSetDO = convert(dataSetReq); DataSetResp dataSetResp = convert(dataSetDO); - conflictCheck(dataSetResp); + // conflictCheck(dataSetResp); updateById(dataSetDO); return dataSetResp; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 976d90fbd..a686d8345 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -225,6 +225,9 @@ public class DatabaseServiceImpl extends ServiceImpl columns = getColumns(modelBuildReq.getDatabaseId(), modelBuildReq.getSql()); + DatabaseResp databaseResp = getDatabase(modelBuildReq.getDatabaseId()); + DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); + columns.forEach(c -> c.setFieldType(engineAdaptor.classifyColumnType(c.getDataType()))); dbColumnMap.put(modelBuildReq.getSql(), columns); } else { for (String table : modelBuildReq.getTables()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 002a365b2..21414c1c8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -6,12 +6,9 @@ import com.alibaba.fastjson.TypeReference; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DataEvent; -import com.tencent.supersonic.common.pojo.DataItem; -import com.tencent.supersonic.common.pojo.ModelRela; -import com.tencent.supersonic.common.pojo.User; +import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; @@ -25,23 +22,14 @@ import com.tencent.supersonic.headless.api.pojo.request.DimValueAliasReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.DatabaseService; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.ModelRelaService; -import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.TagMetaService; +import com.tencent.supersonic.headless.server.service.*; import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; @@ -152,11 +140,7 @@ public class DimensionServiceImpl extends ServiceImpl dimensionDOS = queryDimension(dimensionFilter); return getDataEvent(dimensionDOS, EventType.ADD); @@ -464,13 +448,18 @@ public class DimensionServiceImpl extends ServiceImpl dimensionDOS, EventType eventType) { - List dataItems = dimensionDOS.stream() - .map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE) - .name(dimensionDO.getName()) - .modelId(dimensionDO.getModelId() + Constants.UNDERLINE) - .type(TypeEnums.DIMENSION).build()) - .collect(Collectors.toList()); + List dataItems = + dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList()); return new DataEvent(this, dataItems, eventType); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 0b2548485..936e050b2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -5,59 +5,26 @@ import com.alibaba.fastjson.TypeReference; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DataEvent; -import com.tencent.supersonic.common.pojo.DataItem; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.User; +import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.pojo.enums.*; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.MeasureParam; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; -import com.tencent.supersonic.headless.api.pojo.MetricParam; -import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; -import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq; -import com.tencent.supersonic.headless.api.pojo.request.MetricReq; -import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.request.*; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; -import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; -import com.tencent.supersonic.headless.server.pojo.MetricFilter; -import com.tencent.supersonic.headless.server.pojo.MetricsFilter; -import com.tencent.supersonic.headless.server.pojo.ModelCluster; -import com.tencent.supersonic.headless.server.pojo.ModelFilter; -import com.tencent.supersonic.headless.server.service.CollectService; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.TagMetaService; +import com.tencent.supersonic.headless.server.pojo.*; +import com.tencent.supersonic.headless.server.service.*; import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricConverter; @@ -70,18 +37,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Service @@ -667,12 +623,13 @@ public class MetricServiceImpl extends ServiceImpl } private DataItem getDataItem(MetricDO metricDO) { - MetricResp metricResp = - MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList()); + ModelResp modelResp = modelService.getModel(metricDO.getModelId()); + MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, + ImmutableMap.of(modelResp.getId(), modelResp), Lists.newArrayList()); fillDefaultAgg(metricResp); - return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE) - .name(metricDO.getName()).bizName(metricDO.getBizName()) - .modelId(metricDO.getModelId() + Constants.UNDERLINE).type(TypeEnums.METRIC) + return DataItem.builder().id(metricResp.getId().toString()).name(metricResp.getName()) + .bizName(metricResp.getBizName()).modelId(metricResp.getModelId().toString()) + .domainId(metricResp.getDomainId().toString()).type(TypeEnums.METRIC) .defaultAgg(metricResp.getDefaultAgg()).build(); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index a7da69a61..feaa35e7d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.DbSchema; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.Measure; @@ -78,23 +78,23 @@ import java.util.stream.Collectors; @Slf4j public class ModelServiceImpl implements ModelService { - private ModelRepository modelRepository; + private final ModelRepository modelRepository; - private DatabaseService databaseService; + private final DatabaseService databaseService; - private DimensionService dimensionService; + private final DimensionService dimensionService; - private MetricService metricService; + private final MetricService metricService; - private DomainService domainService; + private final DomainService domainService; - private UserService userService; + private final UserService userService; - private DataSetService dataSetService; + private final DataSetService dataSetService; - private DateInfoRepository dateInfoRepository; + private final DateInfoRepository dateInfoRepository; - private ModelRelaService modelRelaService; + private final ModelRelaService modelRelaService; ExecutorService executor = new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); @@ -292,12 +292,9 @@ public class ModelServiceImpl implements ModelService { if (modelReq.getModelDetail() == null) { return; } - List dims = modelReq.getModelDetail().getDimensions(); + List dims = modelReq.getModelDetail().getDimensions(); List measures = modelReq.getModelDetail().getMeasures(); List identifies = modelReq.getModelDetail().getIdentifiers(); - if (CollectionUtils.isEmpty(dims)) { - throw new InvalidArgumentException("缺少维度信息"); - } for (Measure measure : measures) { String measureForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(measure.getName()); @@ -308,7 +305,7 @@ public class ModelServiceImpl implements ModelService { throw new InvalidArgumentException(message); } } - for (Dim dim : dims) { + for (Dimension dim : dims) { String dimForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dim.getName()); if (StringUtils.isNotBlank(dim.getName()) && StringUtils.isNotBlank(dimForbiddenCharacters)) { @@ -337,12 +334,10 @@ public class ModelServiceImpl implements ModelService { Set relations = new HashSet<>(); for (ModelRela modelRela : modelRelas) { if (modelRela.getFromModelId().equals(modelReq.getId())) { - modelRela.getJoinConditions().stream() - .forEach(r -> relations.add(r.getLeftField())); + modelRela.getJoinConditions().forEach(r -> relations.add(r.getLeftField())); } if (modelRela.getToModelId().equals(modelReq.getId())) { - modelRela.getJoinConditions().stream() - .forEach(r -> relations.add(r.getRightField())); + modelRela.getJoinConditions().forEach(r -> relations.add(r.getRightField())); } } if (relations.isEmpty()) { @@ -351,10 +346,10 @@ public class ModelServiceImpl implements ModelService { // any identify in model relation should not be deleted if (modelReq.getModelDetail() == null || CollectionUtils.isEmpty(modelReq.getModelDetail().getIdentifiers())) { - throw new InvalidArgumentException(String.format("模型关联中主键/外键不存在, 请检查")); + throw new InvalidArgumentException("模型关联中主键/外键不存在, 请检查"); } List modelIdentifiers = modelReq.getModelDetail().getIdentifiers().stream() - .map(i -> i.getBizName()).collect(Collectors.toList()); + .map(Identify::getBizName).collect(Collectors.toList()); for (String rela : relations) { if (!modelIdentifiers.contains(rela)) { throw new InvalidArgumentException(String.format("模型关联中主键/外键(%s)不存在, 请检查", rela)); @@ -459,7 +454,7 @@ public class ModelServiceImpl implements ModelService { } ModelFilter modelFilter = new ModelFilter(); modelFilter.setDomainIds(domainIds); - modelFilter.setIncludesDetail(false); + modelFilter.setIncludesDetail(true); List modelResps = getModelList(modelFilter); if (CollectionUtils.isEmpty(modelResps)) { return modelResps; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java index 2f35bcb1a..a40309f69 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.ValueDistribution; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; @@ -93,7 +93,7 @@ public class TagQueryServiceImpl implements TagQueryService { private void correctDateConf(ItemValueReq itemValueReq, TagResp tag, User user) throws Exception { ModelResp model = modelService.getModel(tag.getModelId()); - List timeDimension = model.getTimeDimension(); + List timeDimension = model.getTimeDimension(); if (CollectionUtils.isEmpty(timeDimension)) { itemValueReq.setDateConf(null); return; @@ -112,12 +112,12 @@ public class TagQueryServiceImpl implements TagQueryService { itemValueReq.setDateConf(dateConf); } - private String queryTagDate(Dim dim) { + private String queryTagDate(Dimension dim) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dim.getDateFormat()); return LocalDate.now().plusDays(-dayBefore).format(formatter); } - private String queryTagDateFromDbBySql(Dim dim, TagResp tag, ItemValueReq itemValueReq, + private String queryTagDateFromDbBySql(Dimension dim, TagResp tag, ItemValueReq itemValueReq, User user) { String sqlPattern = "select max(%s) as %s from tbl where %s is not null"; @@ -129,7 +129,7 @@ public class TagQueryServiceImpl implements TagQueryService { if (Objects.nonNull(itemValueReq) && itemValueReq.getDateConf().getUnit() > 1) { ModelResp model = modelService.getModel(tag.getModelId()); if (Objects.nonNull(model)) { - List timeDims = model.getTimeDimension(); + List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { String dateFormat = timeDims.get(0).getDateFormat(); if (StringUtils.isEmpty(dateFormat)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java index 52610b9ce..2b4dce434 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java @@ -69,7 +69,7 @@ public class MetaEmbeddingTask implements CommandLineRunner { embeddingService.addQuery(embeddingConfig.getMetaCollectionName(), TextSegmentConvert.convertToEmbedding(metricDataItems)); - List dimensionDataItems = dimensionService.getDataEvent().getDataItems(); + List dimensionDataItems = dimensionService.getAllDataEvents().getDataItems(); embeddingService.addQuery(embeddingConfig.getMetaCollectionName(), TextSegmentConvert.convertToEmbedding(dimensionDataItems)); } catch (Exception e) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index e3d76bf97..464372467 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -14,7 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; @@ -401,7 +401,7 @@ public class DictUtils { private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model, Integer itemValueDateStart, Integer itemValueDateEnd) { if (Objects.nonNull(model)) { - List timeDims = model.getTimeDimension(); + List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { DateConf dateConf = new DateConf(); dateConf.setDateMode(DateConf.DateMode.BETWEEN); @@ -496,7 +496,7 @@ public class DictUtils { private boolean partitionedModel(Long modelId) { ModelResp model = modelService.getModel(modelId); if (Objects.nonNull(model)) { - List timeDims = model.getTimeDimension(); + List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { return true; } @@ -507,7 +507,7 @@ public class DictUtils { private String generateDictDateFilterRecent(DictItemResp dictItemResp) { ModelResp model = modelService.getModel(dictItemResp.getModelId()); if (Objects.nonNull(model)) { - List timeDims = model.getTimeDimension(); + List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { String dateFormat = timeDims.get(0).getDateFormat(); if (StringUtils.isEmpty(dateFormat)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index 3e5244898..a5e17be93 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -105,6 +105,8 @@ public class DimensionConverter { dimensionResp.setType(getType(dimensionDO.getType())); dimensionResp.setTypeEnum(TypeEnums.DIMENSION); dimensionResp.setIsTag(dimensionDO.getIsTag()); + dimensionResp.setDomainId(modelRespMap + .getOrDefault(dimensionResp.getModelId(), new ModelResp()).getDomainId()); return dimensionResp; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index e53753818..a04552ea3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.PublishEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; @@ -82,7 +82,7 @@ public class MetricConverter { metricResp.setModelName(modelResp.getName()); metricResp.setModelBizName(modelResp.getBizName()); metricResp.setDomainId(modelResp.getDomainId()); - List timeDims = modelResp.getTimeDimension(); + List timeDims = modelResp.getTimeDimension(); if (CollectionUtils.isNotEmpty(timeDims)) { metricResp.setContainsPartitionDimensions(true); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index d004c9a3b..dcdfe42dc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.ColumnSchema; -import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; @@ -111,7 +111,7 @@ public class ModelConverter { return measureResp; } - public static DimensionReq convert(Dim dim, ModelDO modelDO) { + public static DimensionReq convert(Dimension dim, ModelDO modelDO) { DimensionReq dimensionReq = new DimensionReq(); dimensionReq.setName(dim.getName()); dimensionReq.setBizName(dim.getBizName()); @@ -129,7 +129,6 @@ public class ModelConverter { dimensionReq.setType(dim.getType().name()); dimensionReq .setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); - dimensionReq.setIsTag(dim.getIsTag()); dimensionReq.setTypeParams(dim.getTypeParams()); return dimensionReq; } @@ -165,8 +164,8 @@ public class ModelConverter { public static ModelReq convert(ModelSchema modelSchema, ModelBuildReq modelBuildReq, String tableName) { ModelReq modelReq = new ModelReq(); - modelReq.setName(modelSchema.getName()); - modelReq.setBizName(modelSchema.getBizName()); + modelReq.setName(modelBuildReq.getName()); + modelReq.setBizName(modelBuildReq.getBizName()); modelReq.setDatabaseId(modelBuildReq.getDatabaseId()); modelReq.setDomainId(modelBuildReq.getDomainId()); ModelDetail modelDetail = new ModelDetail(); @@ -188,7 +187,7 @@ public class ModelConverter { columnSchema.getAgg().getOperator(), 1); modelDetail.getMeasures().add(measure); } else { - Dim dim = new Dim(columnSchema.getName(), columnSchema.getColumnName(), + Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(), DimensionType.valueOf(columnSchema.getFiledType().name()), 1); modelDetail.getDimensions().add(dim); } @@ -198,10 +197,12 @@ public class ModelConverter { } private static IdentifyType getIdentifyType(FieldType fieldType) { - if (FieldType.foreign_key.equals(fieldType) || FieldType.primary_key.equals(fieldType)) { + if (FieldType.primary_key.equals(fieldType)) { return IdentifyType.primary; - } else { + } else if (FieldType.foreign_key.equals(fieldType)) { return IdentifyType.foreign; + } else { + return null; } } @@ -214,7 +215,7 @@ public class ModelConverter { return modelDescs; } - private static boolean isCreateDimension(Dim dim) { + private static boolean isCreateDimension(Dimension dim) { return dim.getIsCreateDimension() == 1 && StringUtils.isNotBlank(dim.getName()); } @@ -226,7 +227,7 @@ public class ModelConverter { return measure.getIsCreateMetric() == 1 && StringUtils.isNotBlank(measure.getName()); } - public static List getDimToCreateDimension(ModelDetail modelDetail) { + public static List getDimToCreateDimension(ModelDetail modelDetail) { if (CollectionUtils.isEmpty(modelDetail.getDimensions())) { return Lists.newArrayList(); } @@ -254,7 +255,7 @@ public class ModelConverter { List dimensionReqs = Lists.newArrayList(); ModelDetail modelDetail = JSONObject.parseObject(modelDO.getModelDetail(), ModelDetail.class); - List dims = getDimToCreateDimension(modelDetail); + List dims = getDimToCreateDimension(modelDetail); if (!CollectionUtils.isEmpty(dims)) { dimensionReqs = dims.stream().filter(dim -> StringUtils.isNotBlank(dim.getName())) .map(dim -> convert(dim, modelDO)).collect(Collectors.toList()); diff --git a/headless/server/src/main/resources/mapper/StatMapper.xml b/headless/server/src/main/resources/mapper/StatMapper.xml index 9ed8b8cdf..845625898 100644 --- a/headless/server/src/main/resources/mapper/StatMapper.xml +++ b/headless/server/src/main/resources/mapper/StatMapper.xml @@ -40,28 +40,4 @@ - - \ No newline at end of file diff --git a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index daae81cee..6f903887e 100644 --- a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -116,63 +116,6 @@ - - - - select t.*, (case when t1.id is not null then 1 else 0 end) as isTag - from s2_metric t - left join ( - select * - from s2_tag - where type = 'METRIC' - ) t1 on t.id = t1.item_id - where t.status != 3 - - and t.type = #{type} - - - and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.classifications like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.created_by like CONCAT('%',#{key , jdbcType=VARCHAR},'%') ) - - - and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%') - - - and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%') - - - and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%') - - - and t.sensitive_level = #{sensitiveLevel} - - - and t.status = #{status} - - - and t.model_id in - - #{model} - - - - and t.id in - - #{id} - - - - and t.created_by = #{createdBy} - - - and t1.id is not null - - - and t1.id is null - - - and (t.created_by = #{userName} or t.is_publish = 1) - - -