From 9600456baef4fbaddba3652ac3d0b129296b9c88 Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Mon, 5 Feb 2024 20:30:57 +0800 Subject: [PATCH] headless integrates knowledge (#722) --- .../chat/api/pojo/SchemaElementMatch.java | 1 + .../chat/api/pojo/SemanticParseInfo.java | 1 + .../chat/api/pojo/SemanticSchema.java | 2 + .../supersonic/chat/api/pojo/ViewSchema.java | 2 + .../api/pojo/request/DictLatestTaskReq.java | 23 -- .../api/pojo/request/DictTaskFilterReq.java | 20 - .../chat/api/pojo/request/QueryDataReq.java | 2 +- .../response/ChatDefaultRichConfigResp.java | 2 +- .../api/pojo/response/EntityRichInfoResp.java | 2 +- .../chat/api/pojo/response/QueryResult.java | 2 +- .../chat/api/pojo/response/RecommendResp.java | 2 +- .../chat/api/pojo/response/SearchResult.java | 2 +- chat/core/pom.xml | 64 ---- ...leConfig.java => ChatLocalFileConfig.java} | 6 +- .../core/corrector/BaseSemanticCorrector.java | 2 +- .../chat/core/corrector/WhereCorrector.java | 4 +- .../chat/core/knowledge/DictConfig.java | 13 - .../core/knowledge/DimValue2DictCommand.java | 18 - .../chat/core/knowledge/DimValueDictInfo.java | 31 -- .../chat/core/knowledge/DimValueInfo.java | 26 -- .../chat/core/mapper/BaseMapper.java | 4 +- .../chat/core/mapper/BaseMatchStrategy.java | 24 +- .../core/mapper/DatabaseMatchStrategy.java | 8 +- .../chat/core/mapper/EmbeddingMapper.java | 12 +- .../core/mapper/EmbeddingMatchStrategy.java | 2 +- .../chat/core/mapper/EntityMapper.java | 4 +- .../core/mapper/HanlpDictMatchStrategy.java | 13 +- .../chat/core/mapper/KeywordMapper.java | 20 +- .../chat/core/mapper/MapperHelper.java | 8 +- .../chat/core/mapper/MatchStrategy.java | 5 +- .../core/mapper/ModelWithSemanticType.java | 2 +- .../chat/core/mapper/QueryFilterMapper.java | 6 +- .../chat/core/mapper/SearchMatchStrategy.java | 12 +- .../chat/core/parser/QueryTypeParser.java | 2 +- .../chat/core/parser/plugin/PluginParser.java | 2 +- .../parser/sql/llm/HeuristicViewResolver.java | 2 +- .../parser/sql/llm/LLMRequestService.java | 6 +- .../parser/sql/rule/ContextInheritParser.java | 2 +- .../chat/core/plugin/PluginManager.java | 4 +- .../chat/core/query/BaseSemanticQuery.java | 2 +- .../llm/analytics/MetricAnalyzeQuery.java | 6 +- .../core/query/llm/s2sql/LLMSqlQuery.java | 2 +- .../query/plugin/PluginSemanticQuery.java | 2 +- .../chat/core/query/rule/QueryMatcher.java | 2 +- .../core/query/rule/RuleSemanticQuery.java | 6 +- .../query/rule/metric/MetricFilterQuery.java | 2 +- .../query/rule/metric/MetricGroupByQuery.java | 4 +- .../query/rule/metric/MetricModelQuery.java | 2 +- .../rule/metric/MetricSemanticQuery.java | 4 +- .../query/rule/metric/MetricTagQuery.java | 4 +- .../query/rule/metric/MetricTopNQuery.java | 6 +- .../core/query/rule/tag/TagDetailQuery.java | 4 +- .../core/query/rule/tag/TagFilterQuery.java | 2 +- .../chat/core/query/rule/tag/TagIdQuery.java | 2 +- .../core/query/rule/tag/TagListQuery.java | 4 +- .../core/query/rule/tag/TagSemanticQuery.java | 2 +- .../semantic/BaseSemanticInterpreter.java | 2 +- .../semantic/LocalSemanticInterpreter.java | 2 +- .../semantic/RemoteSemanticInterpreter.java | 2 +- .../semantic/SemanticInterpreter.java | 2 +- .../semantic/ViewSchemaBuilder.java | 10 +- .../chat/core/utils/ComponentFactory.java | 2 +- .../chat/core/utils/DictQueryHelper.java | 219 ----------- .../chat/core/utils/QueryReqBuilder.java | 2 +- .../chat/core/utils/SchemaMatchHelper.java | 2 +- .../main/resources/META-INF/spring.factories | 2 - .../chat/core/s2sql/LLMSqlParserTest.java | 4 +- .../listener/ApplicationStartedListener.java | 6 +- .../listener/SchemaDictUpdateListener.java | 4 +- .../persistence/mapper/DictConfMapper.java | 17 - .../persistence/mapper/DictTaskMapper.java | 18 - .../repository/DictRepository.java | 19 - .../repository/impl/DictRepositoryImpl.java | 83 ---- .../execute/DimensionRecommendProcessor.java | 4 +- .../execute/MetricRatioProcessor.java | 4 +- .../execute/MetricRecommendProcessor.java | 4 +- .../processor/parse/MetricCheckProcessor.java | 0 .../processor/parse/ParseInfoProcessor.java | 2 +- .../server/rest/ChatConfigController.java | 2 +- .../chat/server/rest/ChatQueryController.java | 2 +- .../chat/server/rest/KnowledgeController.java | 114 ------ .../chat/server/service/KnowledgeService.java | 14 - .../server/service/KnowledgeTaskService.java | 24 -- .../chat/server/service/QueryService.java | 2 +- .../chat/server/service/SemanticService.java | 6 +- .../impl/ChatKnowledgeTaskServiceImpl.java | 264 ------------- .../service/impl/ConfigServiceImpl.java | 4 +- .../service/impl/KnowledgeServiceImpl.java | 54 --- .../server/service/impl/QueryServiceImpl.java | 16 +- .../service/impl/RecommendServiceImpl.java | 4 +- .../server/service/impl/SchemaService.java | 2 +- .../service/impl/SearchServiceImpl.java | 24 +- .../chat/server/service/impl/WordService.java | 8 +- .../chat/server/util/ChatConfigHelper.java | 2 +- .../chat/server/util/ComponentFactory.java | 2 +- .../chat/server/util/DictMetaHelper.java | 247 ------------ .../chat/server/util/DictTaskConverter.java | 46 --- .../main/resources/mapper/DictConfMapper.xml | 53 --- .../main/resources/mapper/DictTaskMapper.xml | 75 ---- .../processor/MetricCheckProcessorTest.java | 0 .../context/SemanticParseObjectHelper.java | 2 +- common/pom.xml | 5 + .../com/hankcs/hanlp}/LoadRemoveService.java | 32 +- .../collection/trie/bintrie/BaseNode.java | 15 +- .../hanlp/dictionary/CoreDictionary.java | 22 +- .../hankcs/hanlp/seg/WordBasedSegment.java | 9 +- .../com/hankcs/hanlp/seg/common/Term.java | 15 +- .../headless/api/pojo/ItemValueConfig.java | 20 + .../api/pojo/RelatedSchemaElement.java | 2 +- .../headless}/api/pojo/SchemaElement.java | 2 +- .../headless}/api/pojo/SchemaElementType.java | 2 +- .../headless}/api/pojo/SchemaValueMap.java | 2 +- .../api/pojo/request/DictItemFilter.java | 18 + .../api/pojo/request/DictItemReq.java | 27 ++ .../api/pojo/request/DictSingleTaskReq.java | 16 + .../api/pojo/request/DimensionValueReq.java | 2 +- .../api/pojo/response/DictItemResp.java | 41 ++ .../api/pojo/response/DictTaskResp.java | 20 + .../headless/api/pojo/response/S2Term.java | 50 +++ headless/core/pom.xml | 38 ++ .../headless/core/file}/FileHandler.java | 3 +- .../headless/core/file/FileHandlerImpl.java | 27 +- .../headless/core/file/LocalFileConfig.java | 33 ++ .../core/knowledge/DatabaseMapResult.java | 4 +- .../core/knowledge/DictUpdateMode.java | 2 +- .../headless}/core/knowledge/DictWord.java | 2 +- .../knowledge/DictionaryAttributeUtil.java | 2 +- .../core/knowledge/EmbeddingResult.java | 2 +- .../core/knowledge/HadoopFileIOAdapter.java | 2 +- .../core/knowledge/HanlpMapResult.java | 2 +- .../headless}/core/knowledge/MapResult.java | 2 +- .../core/knowledge/MultiCustomDictionary.java | 9 +- .../core/knowledge/SearchService.java | 30 +- .../core/knowledge/ViewInfoStat.java | 2 +- .../knowledge/builder/BaseWordBuilder.java | 6 +- .../builder/DimensionWordBuilder.java | 6 +- .../knowledge/builder/EntityWordBuilder.java | 6 +- .../knowledge/builder/MetricWordBuilder.java | 6 +- .../knowledge/builder/ModelWordBuilder.java | 6 +- .../knowledge/builder/ValueWordBuilder.java | 6 +- .../knowledge/builder/WordBuilderFactory.java | 2 +- .../core/knowledge/helper}/FileHelper.java | 2 +- .../core/knowledge/helper}/HanlpHelper.java | 38 +- .../knowledge/helper}/HdfsFileHelper.java | 5 +- .../core/knowledge/helper}/NatureHelper.java | 24 +- .../main/resources/META-INF/spring.factories | 2 + headless/server/pom.xml | 5 +- .../persistence/dataobject/DictConfDO.java | 22 ++ .../persistence/dataobject/DictTaskDO.java | 25 ++ .../persistence/mapper/DictConfMapper.java | 9 + .../persistence/mapper/DictTaskMapper.java | 11 + .../repository/DictRepository.java | 29 ++ .../repository/impl/DictRepositoryImpl.java | 142 +++++++ .../server/rest/KnowledgeController.java | 132 +++++++ .../server/service/DictConfService.java | 20 + .../server/service/DictTaskService.java | 18 + .../server/service/KnowledgeService.java | 25 ++ .../service/impl/DictConfServiceImpl.java | 50 +++ .../service/impl/DictTaskServiceImpl.java | 150 ++++++++ .../service/impl/KnowledgeServiceImpl.java | 107 ++++++ .../headless/server/utils/DictUtils.java | 356 ++++++++++++++++++ .../main/resources/META-INF/spring.factories | 4 +- .../src/main/resources/db/chat-schema-h2.sql | 34 +- .../tencent/supersonic/ChatDemoLoader.java | 9 +- .../main/resources/META-INF/spring.factories | 4 +- .../src/main/resources/db/schema-h2.sql | 23 +- .../src/main/resources/db/schema-mysql.sql | 56 ++- .../com/tencent/supersonic/chat/BaseTest.java | 2 +- .../com/tencent/supersonic/chat/TagTest.java | 2 +- .../supersonic/chat/mapper/MapperTest.java | 2 +- .../tencent/supersonic/headless/DictTest.java | 94 +++++ .../tencent/supersonic/util/DataUtils.java | 2 +- .../test/resources/META-INF/spring.factories | 4 +- .../src/test/resources/db/schema-h2.sql | 23 +- 174 files changed, 1908 insertions(+), 1817 deletions(-) delete mode 100644 chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java delete mode 100644 chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java rename chat/core/src/main/java/com/tencent/supersonic/chat/core/config/{LocalFileConfig.java => ChatLocalFileConfig.java} (89%) delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictConfig.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValue2DictCommand.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueDictInfo.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueInfo.java rename chat/core/src/main/java/com/tencent/supersonic/chat/core/{knowledge => query}/semantic/BaseSemanticInterpreter.java (97%) rename chat/core/src/main/java/com/tencent/supersonic/chat/core/{knowledge => query}/semantic/LocalSemanticInterpreter.java (98%) rename chat/core/src/main/java/com/tencent/supersonic/chat/core/{knowledge => query}/semantic/RemoteSemanticInterpreter.java (99%) rename chat/core/src/main/java/com/tencent/supersonic/chat/core/{knowledge => query}/semantic/SemanticInterpreter.java (97%) rename chat/core/src/main/java/com/tencent/supersonic/chat/core/{knowledge => query}/semantic/ViewSchemaBuilder.java (95%) delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java delete mode 100644 chat/core/src/main/resources/META-INF/spring.factories delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictConfMapper.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictTaskMapper.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/DictRepository.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/DictRepositoryImpl.java create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/KnowledgeController.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeService.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeTaskService.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatKnowledgeTaskServiceImpl.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/KnowledgeServiceImpl.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictTaskConverter.java delete mode 100644 chat/server/src/main/resources/mapper/DictConfMapper.xml delete mode 100644 chat/server/src/main/resources/mapper/DictTaskMapper.xml create mode 100644 chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java rename {chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge => common/src/main/java/com/hankcs/hanlp}/LoadRemoveService.java (64%) rename {chat/core => common}/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java (92%) rename {chat/core => common}/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java (93%) rename {chat/core => common}/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java (98%) rename {chat/core => common}/src/main/java/com/hankcs/hanlp/seg/common/Term.java (80%) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java rename {chat/api/src/main/java/com/tencent/supersonic/chat => headless/api/src/main/java/com/tencent/supersonic/headless}/api/pojo/RelatedSchemaElement.java (84%) rename {chat/api/src/main/java/com/tencent/supersonic/chat => headless/api/src/main/java/com/tencent/supersonic/headless}/api/pojo/SchemaElement.java (96%) rename {chat/api/src/main/java/com/tencent/supersonic/chat => headless/api/src/main/java/com/tencent/supersonic/headless}/api/pojo/SchemaElementType.java (70%) rename {chat/api/src/main/java/com/tencent/supersonic/chat => headless/api/src/main/java/com/tencent/supersonic/headless}/api/pojo/SchemaValueMap.java (88%) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java rename {chat/api/src/main/java/com/tencent/supersonic/chat => headless/api/src/main/java/com/tencent/supersonic/headless}/api/pojo/request/DimensionValueReq.java (82%) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java rename {chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge => headless/core/src/main/java/com/tencent/supersonic/headless/core/file}/FileHandler.java (95%) rename chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LocalFileHandler.java => headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandlerImpl.java (93%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/file/LocalFileConfig.java rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/DatabaseMapResult.java (84%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/DictUpdateMode.java (91%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/DictWord.java (92%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/DictionaryAttributeUtil.java (96%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/EmbeddingResult.java (92%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/HadoopFileIOAdapter.java (94%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/HanlpMapResult.java (94%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/MapResult.java (78%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/MultiCustomDictionary.java (98%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/SearchService.java (87%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/ViewInfoStat.java (85%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/BaseWordBuilder.java (83%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/DimensionWordBuilder.java (92%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/EntityWordBuilder.java (86%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/MetricWordBuilder.java (92%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/ModelWordBuilder.java (86%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/ValueWordBuilder.java (86%) rename {chat/core/src/main/java/com/tencent/supersonic/chat => headless/core/src/main/java/com/tencent/supersonic/headless}/core/knowledge/builder/WordBuilderFactory.java (92%) rename {chat/core/src/main/java/com/tencent/supersonic/chat/core/utils => headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper}/FileHelper.java (97%) rename {chat/core/src/main/java/com/tencent/supersonic/chat/core/utils => headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper}/HanlpHelper.java (88%) rename {chat/core/src/main/java/com/tencent/supersonic/chat/core/utils => headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper}/HdfsFileHelper.java (95%) rename {chat/core/src/main/java/com/tencent/supersonic/chat/core/utils => headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper}/NatureHelper.java (88%) create mode 100644 headless/core/src/main/resources/META-INF/spring.factories create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DictRepository.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java create mode 100644 launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java index 0062988e3..b205c87cb 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.chat.api.pojo; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 36e146d29..da7992911 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.FilterType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.Data; import java.util.ArrayList; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index db103110f..92434bf47 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.chat.api.pojo; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import org.springframework.util.CollectionUtils; import java.io.Serializable; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java index 9c6c88a04..fd71a4847 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java @@ -1,6 +1,8 @@ package com.tencent.supersonic.chat.api.pojo; import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java deleted file mode 100644 index f38167c13..000000000 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.supersonic.chat.api.pojo.request; - -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; - -import javax.validation.constraints.NotNull; -import java.util.List; - -import static java.time.LocalDate.now; - -@ToString -@Data -@NoArgsConstructor -public class DictLatestTaskReq { - - @NotNull - private Long modelId; - - private List dimIds; - - private String createdAt = now().plusDays(-4).toString(); -} \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java deleted file mode 100644 index d58364ff3..000000000 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.tencent.supersonic.chat.api.pojo.request; - -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import lombok.Data; -import lombok.ToString; - -@ToString -@Data -public class DictTaskFilterReq { - - private Long id; - - private String name; - - private String createdBy; - - private String createdAt; - - private TaskStatusEnum status; -} \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java index b11ce0ce0..5091bc6a2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.common.pojo.DateConf; import java.util.HashSet; import java.util.Set; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java index 82dc7c5d3..53bd9222e 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeMode; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java index a8610fed1..b4731532e 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import java.util.List; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java index 44e5ec691..4cce89735 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.common.pojo.QueryAuthorization; import com.tencent.supersonic.common.pojo.QueryColumn; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java index 8490ab549..a151623a6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.Data; import java.util.List; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java index e2426cfbb..4e7b251de 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import java.util.Objects; import lombok.Builder; import lombok.Data; diff --git a/chat/core/pom.xml b/chat/core/pom.xml index 924b242f7..2621995b7 100644 --- a/chat/core/pom.xml +++ b/chat/core/pom.xml @@ -21,70 +21,6 @@ org.springframework spring-context - - - com.hankcs - hanlp - ${hanlp.version} - - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - - - org.apache.zookeeper - zookeeper - - - javax.servlet - servlet-api - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - org.apache.zookeeper - zookeeper - - - javax.servlet - servlet-api - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - org.apache.zookeeper - zookeeper - - - org.apache.curator - * - - - javax.servlet - servlet-api - - - org.testng testng diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LocalFileConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java similarity index 89% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LocalFileConfig.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java index b039c3924..14023a691 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LocalFileConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.chat.core.config; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; + import java.io.FileNotFoundException; + import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -11,7 +13,7 @@ import org.springframework.context.annotation.Configuration; @Data @Configuration @Slf4j -public class LocalFileConfig { +public class ChatLocalFileConfig { @Value("${dict.directory.latest:/data/dictionary/custom}") diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java index 80c701b19..2f43864c9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java index dc9fb9775..7c471f320 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictConfig.java deleted file mode 100644 index 5f1f8cd47..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictConfig.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.tencent.supersonic.chat.core.knowledge; - -import java.util.List; -import lombok.Data; - - -@Data -public class DictConfig { - - private Long modelId; - - private List dimValueInfoList; -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValue2DictCommand.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValue2DictCommand.java deleted file mode 100644 index f9a865692..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValue2DictCommand.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tencent.supersonic.chat.core.knowledge; - - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import lombok.Data; - -@Data -public class DimValue2DictCommand { - - private DictUpdateMode updateMode; - - private List modelIds; - - private Map> modelAndDimPair = new HashMap<>(); -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueDictInfo.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueDictInfo.java deleted file mode 100644 index f5c374afb..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueDictInfo.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.tencent.supersonic.chat.core.knowledge; - - -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; - -import java.util.Date; -import java.util.Set; - -import lombok.Data; - -@Data -public class DimValueDictInfo { - - private Long id; - - private String name; - - private String description; - - private String command; - - private TaskStatusEnum status; - - private String createdBy; - - private Date createdAt; - - private Long elapsedMs; - - private Set dimIds; -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueInfo.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueInfo.java deleted file mode 100644 index 83cf7acbb..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueInfo.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.tencent.supersonic.chat.core.knowledge; - - -import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import java.util.List; -import javax.validation.constraints.NotNull; - -public class DimValueInfo { - - /** - * metricId、DimensionId、domainId - */ - private Long itemId; - - /** - * type: IntentionTypeEnum - * temporarily only supports dimension-related information - */ - @NotNull - private TypeEnums type = TypeEnums.DIMENSION; - - private List blackList; - private List whiteList; - private List ruleList; - private Boolean isDictInfo; -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java index dc0f98298..3a79e29b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.core.mapper; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.ViewSchema; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java index 887e33deb..af9a49a58 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.core.mapper; -import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.utils.NatureHelper; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -28,7 +28,8 @@ public abstract class BaseMatchStrategy implements MatchStrategy { private MapperHelper mapperHelper; @Override - public Map> match(QueryContext queryContext, List terms, Set detectViewIds) { + public Map> match(QueryContext queryContext, List terms, + Set detectViewIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; @@ -43,7 +44,7 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return result; } - public List detect(QueryContext queryContext, List terms, Set detectModelIds) { + public List detect(QueryContext queryContext, List terms, Set detectModelIds) { Map regOffsetToLength = getRegOffsetToLength(terms); String text = queryContext.getQueryText(); Set results = new HashSet<>(); @@ -72,9 +73,10 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return; } - public Map getRegOffsetToLength(List terms) { - return terms.stream().sorted(Comparator.comparing(Term::length)) - .collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2)); + public Map getRegOffsetToLength(List terms) { + return terms.stream().sorted(Comparator.comparing(S2Term::length)) + .collect(Collectors.toMap(S2Term::getOffset, term -> term.word.length(), + (value1, value2) -> value2)); } public void selectResultInOneRound(Set existResults, List oneRoundResults) { @@ -102,7 +104,7 @@ public abstract class BaseMatchStrategy implements MatchStrategy { } } - public List getMatches(QueryContext queryContext, List terms) { + public List getMatches(QueryContext queryContext, List terms) { Set viewIds = mapperHelper.getViewIds(queryContext.getViewId(), queryContext.getAgent()); terms = filterByViewId(terms, viewIds); Map> matchResult = match(queryContext, terms, viewIds); @@ -120,7 +122,7 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return matches; } - public List filterByViewId(List terms, Set viewIds) { + public List filterByViewId(List terms, Set viewIds) { logTerms(terms); if (CollectionUtils.isNotEmpty(viewIds)) { terms = terms.stream().filter(term -> { @@ -136,11 +138,11 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return terms; } - public void logTerms(List terms) { + public void logTerms(List terms) { if (CollectionUtils.isEmpty(terms)) { return; } - for (Term term : terms) { + for (S2Term term : terms) { log.debug("word:{},nature:{},frequency:{}", term.word, term.nature.toString(), term.getFrequency()); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java index 3be851f87..27726248f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat.core.mapper; -import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import lombok.extern.slf4j.Slf4j; @@ -36,7 +36,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy private List allElements; @Override - public Map> match(QueryContext queryContext, List terms, + public Map> match(QueryContext queryContext, List terms, Set detectModelIds) { this.allElements = getSchemaElements(queryContext); return super.match(queryContext, terms, detectModelIds); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java index 8b658f38f..8182c4c8b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.chat.core.mapper; import com.alibaba.fastjson.JSONObject; -import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult; -import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult; +import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.embedding.Retrieval; import java.util.List; @@ -26,7 +26,7 @@ public class EmbeddingMapper extends BaseMapper { public void doMap(QueryContext queryContext) { //1. query from embedding by queryText String queryText = queryContext.getQueryText(); - List terms = HanlpHelper.getTerms(queryText); + List terms = HanlpHelper.getTerms(queryText); EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class); List matchResults = matchStrategy.getMatches(queryContext, terms); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java index 23feceaae..a4461f257 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.mapper; import com.google.common.collect.Lists; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult; +import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.pojo.Constants; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java index 0e9fd87a9..53d8560a4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.core.mapper; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.ViewSchema; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java index 6757447c9..923b1d157 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.core.mapper; -import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.chat.core.knowledge.SearchService; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import java.util.HashMap; @@ -35,7 +35,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { private OptimizationConfig optimizationConfig; @Override - public Map> match(QueryContext queryContext, List terms, + public Map> match(QueryContext queryContext, List terms, Set detectModelIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { @@ -60,16 +60,15 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { public void detectByStep(QueryContext queryContext, Set existResults, Set detectModelIds, Integer startIndex, Integer index, int offset) { String text = queryContext.getQueryText(); - Integer agentId = queryContext.getAgentId(); String detectSegment = text.substring(startIndex, index); // step1. pre search Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize(); LinkedHashSet hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize, - agentId, detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); + detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); // step2. suffix search LinkedHashSet suffixHanlpMapResults = SearchService.suffixSearch(detectSegment, - oneDetectionMaxSize, agentId, detectModelIds).stream() + oneDetectionMaxSize, detectModelIds).stream() .collect(Collectors.toCollection(LinkedHashSet::new)); hanlpMapResults.addAll(suffixHanlpMapResults); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java index 33aa4772e..167af8d8f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java @@ -1,15 +1,16 @@ package com.tencent.supersonic.chat.core.mapper; -import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult; -import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; -import com.tencent.supersonic.chat.core.utils.NatureHelper; +import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; import com.tencent.supersonic.common.util.ContextUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -32,7 +33,8 @@ public class KeywordMapper extends BaseMapper { public void doMap(QueryContext queryContext) { String queryText = queryContext.getQueryText(); //1.hanlpDict Match - List terms = HanlpHelper.getTerms(queryText); + KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class); + List terms = knowledgeService.getTerms(queryText); HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class); List hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms); @@ -46,7 +48,7 @@ public class KeywordMapper extends BaseMapper { } private void convertHanlpMapResultToMapInfo(List mapResults, QueryContext queryContext, - List terms) { + List terms) { if (CollectionUtils.isEmpty(mapResults)) { return; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java index 50a25e4d6..cd4fa5cc4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat.core.mapper; import com.hankcs.hanlp.algorithm.EditDistance; -import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.utils.NatureHelper; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -36,8 +36,8 @@ public class MapperHelper { return index; } - public Integer getStepOffset(List termList, Integer index) { - List offsetList = termList.stream().sorted(Comparator.comparing(Term::getOffset)) + public Integer getStepOffset(List termList, Integer index) { + List offsetList = termList.stream().sorted(Comparator.comparing(S2Term::getOffset)) .map(term -> term.getOffset()).collect(Collectors.toList()); for (int j = 0; j < termList.size() - 1; j++) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java index ff1374b75..6c064df99 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java @@ -1,7 +1,8 @@ package com.tencent.supersonic.chat.core.mapper; -import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; + import java.util.List; import java.util.Map; import java.util.Set; @@ -12,6 +13,6 @@ import java.util.Set; */ public interface MatchStrategy { - Map> match(QueryContext queryContext, List terms, Set detectModelId); + Map> match(QueryContext queryContext, List terms, Set detectModelId); } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java index 102d48e4b..92c734557 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.core.mapper; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import java.io.Serializable; import lombok.Data; import lombok.ToString; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java index d3042c00c..8e054891d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.chat.core.mapper; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder; +import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import lombok.extern.slf4j.Slf4j; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java index a756d63b8..13f264534 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.core.mapper; import com.google.common.collect.Lists; -import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.chat.core.knowledge.SearchService; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.DictWordType; import java.util.List; @@ -26,7 +26,7 @@ public class SearchMatchStrategy extends BaseMatchStrategy { private static final int SEARCH_SIZE = 3; @Override - public Map> match(QueryContext queryContext, List originals, + public Map> match(QueryContext queryContext, List originals, Set detectModelIds) { String text = queryContext.getQueryText(); Map regOffsetToLength = getRegOffsetToLength(originals); @@ -52,9 +52,9 @@ public class SearchMatchStrategy extends BaseMatchStrategy { if (StringUtils.isNotEmpty(detectSegment)) { List hanlpMapResults = SearchService.prefixSearch(detectSegment, - SearchService.SEARCH_SIZE, queryContext.getAgentId(), detectModelIds); + SearchService.SEARCH_SIZE, detectModelIds); List suffixHanlpMapResults = SearchService.suffixSearch( - detectSegment, SEARCH_SIZE, queryContext.getAgentId(), detectModelIds); + detectSegment, SEARCH_SIZE, detectModelIds); hanlpMapResults.addAll(suffixHanlpMapResults); // remove entity name where search hanlpMapResults = hanlpMapResults.stream().filter(entry -> { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java index b716a0e1a..de31c0ef6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.parser; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java index 6d034e5b5..8ff3ef04a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.parser.plugin; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java index ecffef4bd..366d0666e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.SemanticQuery; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java index abc2c7507..3dce92fb3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java @@ -1,16 +1,16 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.agent.AgentToolType; import com.tencent.supersonic.chat.core.agent.NL2SQLTool; import com.tencent.supersonic.chat.core.config.LLMParserConfig; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.parser.SatisfactionChecker; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java index 1097af9b0..6d4f9cf74 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.parser.sql.rule; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.core.parser.SemanticParser; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java index a6963baa5..f9d315c14 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java @@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.core.plugin; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.agent.AgentToolType; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 0a6942b69..03fc03164 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.Aggregator; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java index 8ffbc6e9f..a08bc31b6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java @@ -2,14 +2,14 @@ package com.tencent.supersonic.chat.core.query.llm.analytics; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery; import com.tencent.supersonic.chat.core.utils.ComponentFactory; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java index b6a0b72e0..940082617 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.query.llm.s2sql; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java index 78c54b009..c62b53fa8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.query.plugin; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java index 96e988878..80e0c71a3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.query.rule; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import lombok.Data; import lombok.ToString; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java index 629450078..64159be43 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java @@ -2,16 +2,16 @@ package com.tencent.supersonic.chat.core.query.rule; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.BaseSemanticQuery; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java index bf7b881a2..f5ccf41c4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java index 2eec00f67..8756a7f14 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.core.query.rule.metric; import org.springframework.stereotype.Component; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java index fe6be3b99..7496970a8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java @@ -4,7 +4,7 @@ import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.Optio import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import org.springframework.stereotype.Component; @Component diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java index 4aa69ac4d..c372663e3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.ViewSchema; @@ -29,7 +29,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { @Override public List match(List candidateElementMatches, - QueryContext queryCtx) { + QueryContext queryCtx) { return super.match(candidateElementMatches, queryCtx); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java index c07ace227..b26d7765a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java index ff8bca6f0..ff8df095a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java index 16b97ba78..20794af48 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.query.rule.tag; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java index 60b90c6ef..127f40d54 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.core.query.rule.tag; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java index 60e1c7ed7..d4c83e512 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.core.query.rule.tag; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java index a2c3709a1..c8a43df57 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.query.rule.tag; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.core.pojo.ChatContext; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java index c38a44184..9abcd42e3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java @@ -15,7 +15,7 @@ import java.time.LocalDate; import java.util.List; import java.util.Objects; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java similarity index 97% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java index 33d0d3010..850499945 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge.semantic; +package com.tencent.supersonic.chat.core.query.semantic; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java similarity index 98% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java index dec5c7c71..d8796f215 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge.semantic; +package com.tencent.supersonic.chat.core.query.semantic; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java similarity index 99% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java index 993d4b27e..9114640aa 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge.semantic; +package com.tencent.supersonic.chat.core.query.semantic; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java similarity index 97% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java index 4e210622d..98662aedd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge.semantic; +package com.tencent.supersonic.chat.core.query.semantic; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ViewSchemaBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ViewSchemaBuilder.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java index 289b1863b..44d9522a8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ViewSchemaBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java @@ -1,10 +1,10 @@ -package com.tencent.supersonic.chat.core.knowledge.semantic; +package com.tencent.supersonic.chat.core.query.semantic; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; +import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.RelateDimension; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java index 0e47a8305..cb80644ea 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.core.utils; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.parser.JavaLLMProxy; import com.tencent.supersonic.chat.core.parser.LLMProxy; import com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java deleted file mode 100644 index 21fe9be44..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java +++ /dev/null @@ -1,219 +0,0 @@ -package com.tencent.supersonic.chat.core.utils; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.config.DefaultMetric; -import com.tencent.supersonic.chat.core.config.Dim4Dict; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.StringJoiner; - -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE; - -@Slf4j -@Component -public class DictQueryHelper { - - private static final Long MAX_FREQUENCY = 99999999L; - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - @Value("${dimension.multi.value.split:#}") - private String dimMultiValueSplit; - @Value("${dimension.value.show:50}") - private Integer printDataShow; - @Value("${dimension.max.limit:3000000}") - private Long dimMaxLimit; - @Value("${dimension.white.weight:60000000}") - private Long dimensionWhiteWeight; - - public List fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict, - User user) { - List data = new ArrayList<>(); - QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict); - try { - SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructCmd, user); - - log.info("fetchDimValueSingle sql:{}", semanticQueryResp.getSql()); - String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId()); - String dimNameRewrite = rewriteDimName(semanticQueryResp.getColumns(), dim4Dict.getBizName()); - data = generateFileData(semanticQueryResp.getResultList(), nature, dimNameRewrite, - defaultMetricDesc.getBizName(), dim4Dict); - if (!CollectionUtils.isEmpty(data)) { - int size = (data.size() > printDataShow) ? printDataShow : data.size(); - log.info("data:{}", data.subList(0, size)); - } else { - log.warn("data is empty. nature:{}", nature); - if (Objects.nonNull(semanticQueryResp)) { - log.warn("sql:{}", semanticQueryResp.getSql()); - } - } - - } catch (Exception e) { - log.warn("fetchDimValueSingle,e:", e); - } - return data; - } - - private String rewriteDimName(List columns, String bizName) { - // metric parser join dimension style - String dimNameRewrite = bizName; - - if (!CollectionUtils.isEmpty(columns)) { - for (QueryColumn column : columns) { - if (StringUtils.isNotEmpty(column.getNameEn())) { - String nameEn = column.getNameEn(); - if (nameEn.endsWith(UNDERLINE_DOUBLE + bizName)) { - dimNameRewrite = nameEn; - } - } - } - } - return dimNameRewrite; - } - - private List generateFileData(List> resultList, String nature, String dimName, - String metricName, Dim4Dict dim4Dict) { - List data = new ArrayList<>(); - if (CollectionUtils.isEmpty(resultList)) { - return data; - } - Map valueAndFrequencyPair = new HashMap<>(2000); - for (Map line : resultList) { - - if (CollectionUtils.isEmpty(line) || !line.containsKey(dimName) - || !line.containsKey(metricName) - || line.get(dimName) == null) { - continue; - } - - String dimValue = line.get(dimName).toString(); - Object metricObject = line.get(metricName); - if (Strings.isNotEmpty(dimValue) && Objects.nonNull(metricObject)) { - Long metric = Math.round(Double.parseDouble(metricObject.toString())); - mergeMultivaluedValue(valueAndFrequencyPair, dimValue, metric); - } - - } - constructDataLines(valueAndFrequencyPair, nature, data, dim4Dict); - return data; - } - - private void constructDataLines(Map valueAndFrequencyPair, String nature, - List data, Dim4Dict dim4Dict) { - valueAndFrequencyPair.forEach((dimValue, metric) -> { - if (metric > MAX_FREQUENCY) { - metric = MAX_FREQUENCY; - } - if (Strings.isNotEmpty(dimValue) && dimValue.contains(SPACE)) { - dimValue = dimValue.replace(SPACE, "#"); - } - data.add(String.format("%s %s %s", dimValue, nature, metric)); - }); - - if (Objects.nonNull(dim4Dict) && !CollectionUtils.isEmpty(dim4Dict.getWhiteList())) { - dim4Dict.getWhiteList().stream() - .forEach(white -> data.add(String.format("%s %s %s", white, nature, dimensionWhiteWeight))); - } - } - - private void mergeMultivaluedValue(Map valueAndFrequencyPair, String dimValue, Long metric) { - if (Strings.isEmpty(dimValue)) { - return; - } - Map tmp = new HashMap<>(); - if (dimValue.contains(dimMultiValueSplit)) { - Arrays.stream(dimValue.split(dimMultiValueSplit)) - .forEach(dimValueSingle -> tmp.put(dimValueSingle, metric)); - } else { - tmp.put(dimValue, metric); - } - - for (String value : tmp.keySet()) { - long metricOld = valueAndFrequencyPair.containsKey(value) ? valueAndFrequencyPair.get(value) : 0L; - valueAndFrequencyPair.put(value, metric + metricOld); - } - } - - private QueryStructReq generateQueryStructCmd(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict) { - QueryStructReq queryStructCmd = new QueryStructReq(); - - List filters = generateFilters(dim4Dict, queryStructCmd); - queryStructCmd.setDimensionFilters(filters); - - List aggregators = new ArrayList<>(); - aggregators.add(new Aggregator(defaultMetricDesc.getBizName(), AggOperatorEnum.SUM)); - queryStructCmd.setAggregators(aggregators); - - List orders = new ArrayList<>(); - orders.add(new Order(defaultMetricDesc.getBizName(), Constants.DESC_UPPER)); - queryStructCmd.setOrders(orders); - - DateConf dateInfo = new DateConf(); - dateInfo.setDateMode(DateConf.DateMode.RECENT); - log.debug("defaultMetric unit():{}", defaultMetricDesc.getUnit()); - dateInfo.setUnit(defaultMetricDesc.getUnit()); - dateInfo.setPeriod(defaultMetricDesc.getPeriod()); - queryStructCmd.setDateInfo(dateInfo); - - queryStructCmd.setLimit(dimMaxLimit); - queryStructCmd.setNeedAuth(false); - return queryStructCmd; - - } - - private List generateFilters(Dim4Dict dim4Dict, QueryStructReq queryStructCmd) { - String whereStr = generateFilter(dim4Dict); - if (Strings.isEmpty(whereStr)) { - return new ArrayList<>(); - } - Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, whereStr); - List filters = Objects.isNull(queryStructCmd.getOriginalFilter()) ? new ArrayList<>() - : queryStructCmd.getOriginalFilter(); - filters.add(filter); - return filters; - } - - private String generateFilter(Dim4Dict dim4Dict) { - if (Objects.isNull(dim4Dict)) { - return ""; - } - StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE); - - String dimName = dim4Dict.getBizName(); - if (!CollectionUtils.isEmpty(dim4Dict.getBlackList())) { - StringJoiner joinerBlack = new StringJoiner(COMMA); - dim4Dict.getBlackList().stream().forEach(black -> joinerBlack.add(APOSTROPHE + black + APOSTROPHE)); - joiner.add(String.format("(%s not in (%s))", dimName, joinerBlack.toString())); - } - - if (!CollectionUtils.isEmpty(dim4Dict.getRuleList())) { - dim4Dict.getRuleList().stream().forEach(rule -> joiner.add(rule)); - } - - return joiner.toString(); - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java index 5da9e4f2f..ca3ff8a40 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.utils; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.common.pojo.Aggregator; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java index 3bcf2157f..97380b07e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.utils; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import java.util.List; import java.util.Set; import java.util.stream.Collectors; diff --git a/chat/core/src/main/resources/META-INF/spring.factories b/chat/core/src/main/resources/META-INF/spring.factories deleted file mode 100644 index ca73c1e2b..000000000 --- a/chat/core/src/main/resources/META-INF/spring.factories +++ /dev/null @@ -1,2 +0,0 @@ -com.tencent.supersonic.chat.core.knowledge.FileHandler=\ - com.tencent.supersonic.chat.core.knowledge.LocalFileHandler \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java index eb173d54d..6f356ae1f 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.core.s2sql; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import org.junit.jupiter.api.Test; import org.mockito.Mockito; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java index f7cfe0efe..217d4ec9e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java @@ -1,11 +1,13 @@ package com.tencent.supersonic.chat.server.listener; -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import com.tencent.supersonic.chat.server.service.KnowledgeService; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.chat.server.service.impl.WordService; + import java.util.List; import java.util.concurrent.CompletableFuture; + +import com.tencent.supersonic.headless.server.service.KnowledgeService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java index 5707e3ea0..78c23bbad 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.listener; -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DataEvent; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictConfMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictConfMapper.java deleted file mode 100644 index c45055ff4..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictConfMapper.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.tencent.supersonic.chat.server.persistence.mapper; - - -import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface DictConfMapper { - - Boolean createDictConf(DictConfDO dictConfDO); - - Boolean editDictConf(DictConfDO dictConfDO); - - Boolean upsertDictInfo(DictConfDO dictConfDO); - - DictConfDO getDictInfoByModelId(Long modelId); -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictTaskMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictTaskMapper.java deleted file mode 100644 index ce5e6e66e..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictTaskMapper.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tencent.supersonic.chat.server.persistence.mapper; - -import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO; -import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; - -import java.util.List; - -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface DictTaskMapper { - - Long createDimValueTask(DictTaskDO dictTaskDO); - - Boolean updateTaskStatus(DictTaskDO dictTaskDO); - - List searchDictTaskList(DictTaskFilterReq filter); -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/DictRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/DictRepository.java deleted file mode 100644 index a3748197d..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/DictRepository.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.tencent.supersonic.chat.server.persistence.repository; - - -import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; -import com.tencent.supersonic.chat.core.knowledge.DictConfig; -import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo; -import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO; -import java.util.List; - -public interface DictRepository { - - Long createDimValueDictTask(DictTaskDO dictTaskDO); - - Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO); - - List searchDictTaskList(DictTaskFilterReq filter); - - DictConfig getDictInfoByModelId(Long modelId); -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/DictRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/DictRepositoryImpl.java deleted file mode 100644 index 9e850c909..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/DictRepositoryImpl.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.tencent.supersonic.chat.server.persistence.repository.impl; - -import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; -import com.tencent.supersonic.chat.core.knowledge.DictConfig; -import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo; -import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO; -import com.tencent.supersonic.chat.server.persistence.mapper.DictConfMapper; -import com.tencent.supersonic.chat.server.persistence.mapper.DictTaskMapper; -import com.tencent.supersonic.chat.server.persistence.repository.DictRepository; -import com.tencent.supersonic.chat.server.util.DictTaskConverter; -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.common.util.JsonUtil; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Repository; -import org.springframework.util.CollectionUtils; - - -@Repository -@Slf4j -public class DictRepositoryImpl implements DictRepository { - - private final DictTaskMapper dictTaskMapper; - private final DictConfMapper dictConfMapper; - - public DictRepositoryImpl(DictTaskMapper dictTaskMapper, - DictConfMapper dictConfMapper) { - this.dictTaskMapper = dictTaskMapper; - this.dictConfMapper = dictConfMapper; - } - - @Override - public Long createDimValueDictTask(DictTaskDO dictTaskDO) { - dictTaskMapper.createDimValueTask(dictTaskDO); - return dictTaskDO.getId(); - } - - @Override - public Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO) { - dictTaskDO.setStatus(status); - Date createdAt = dictTaskDO.getCreatedAt(); - long elapsedMs = System.currentTimeMillis() - createdAt.getTime(); - dictTaskDO.setElapsedMs(elapsedMs); - dictTaskMapper.updateTaskStatus(dictTaskDO); - return true; - } - - @Override - public List searchDictTaskList(DictTaskFilterReq filter) { - List dimValueDictDescList = new ArrayList<>(); - log.info("filter:{}", filter); - List dictTaskDOList = dictTaskMapper.searchDictTaskList(filter); - if (!CollectionUtils.isEmpty(dictTaskDOList)) { - dictTaskDOList.stream().forEach(dictTaskDO -> { - DimValueDictInfo dimValueDictDesc = new DimValueDictInfo(); - BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc); - dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus())); - if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) { - Set dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class); - dimValueDictDesc.setDimIds(dimIds); - } - dimValueDictDescList.add(dimValueDictDesc); - }); - } - return dimValueDictDescList; - } - - @Override - public DictConfig getDictInfoByModelId(Long modelId) { - DictConfDO dictConfDO = dictConfMapper.getDictInfoByModelId(modelId); - if (Objects.isNull(dictConfDO)) { - return null; - } - return DictTaskConverter.dictConfPO2Config(dictConfDO); - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index dda9c80d3..84035877f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java index 408c4a561..4be32c2db 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java @@ -11,14 +11,13 @@ import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; import static com.tencent.supersonic.common.pojo.Constants.WEEK; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.core.config.AggregatorConfig; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.DateConf; @@ -29,6 +28,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.RatioOverType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import java.text.DecimalFormat; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java index c91f4deff..7f3330ec6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.alibaba.fastjson.JSONObject; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java new file mode 100644 index 000000000..e69de29bb diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java index db52b0d46..b844974f7 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.processor.parse; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java index 4898b6708..ebada89d4 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java index bdfcc9d63..7c24b5050 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/KnowledgeController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/KnowledgeController.java deleted file mode 100644 index 1c948ece3..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/KnowledgeController.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.tencent.supersonic.chat.server.rest; - - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq; -import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; -import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp; -import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand; -import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo; -import com.tencent.supersonic.chat.server.listener.ApplicationStartedListener; -import com.tencent.supersonic.chat.server.service.KnowledgeTaskService; -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - - -@RestController -@RequestMapping("/api/chat/dict") -public class KnowledgeController { - - @Autowired - private KnowledgeTaskService knowledgeTaskService; - - @Autowired - private ApplicationStartedListener applicationStartedListener; - - /** - * addDictInfo - * write specific dimension values to the knowledge base - * - * @param dimValue2DictCommend - */ - @PostMapping("/task") - public Long addDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return knowledgeTaskService.addDictTask(dimValue2DictCommend, user); - } - - /** - * deleteDictInfo - * remove specific dimension values from the knowledge base - * - * @param dimValue2DictCommend - */ - @PostMapping("/task/delete") - public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return knowledgeTaskService.deleteDictTask(dimValue2DictCommend, user); - } - - /** - * searchDictTaskList - * - * @param filter - */ - @PostMapping("/task/search") - public List searchDictTaskList(@RequestBody DictTaskFilterReq filter, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return knowledgeTaskService.searchDictTaskList(filter, user); - } - - /** - * searchDictLatestTaskList - */ - @PostMapping("/task/search/latest") - public List searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return knowledgeTaskService.searchDictLatestTaskList(filter, user); - } - - /** - * getDictRootPath - * get knowledge base file directory - * - * @return - */ - @GetMapping("/rootPath") - public String getDictRootPath(HttpServletRequest request, - HttpServletResponse response) { - return knowledgeTaskService.getDictRootPath(); - } - - /** - * updateDimValue - * update in-memory dictionary files in real time - * - * @param request - * @param response - * @return - */ - @PutMapping("/knowledge/dimValue") - public Boolean updateDimValue(HttpServletRequest request, - HttpServletResponse response) { - return applicationStartedListener.updateKnowledgeDimValue(); - } - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeService.java deleted file mode 100644 index 9ebb1a6a2..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeService.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.chat.server.service; - -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import java.util.List; - -public interface KnowledgeService { - - void updateSemanticKnowledge(List natures); - - void reloadAllData(List natures); - - void updateOnlineKnowledge(List natures); - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeTaskService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeTaskService.java deleted file mode 100644 index dbf27d522..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeTaskService.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.tencent.supersonic.chat.server.service; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq; -import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; -import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp; -import com.tencent.supersonic.chat.core.knowledge.DictConfig; -import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand; -import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo; -import java.util.List; - -public interface KnowledgeTaskService { - Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user); - - Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user); - - List searchDictTaskList(DictTaskFilterReq filter, User user); - - DictConfig getDictInfoByModelId(Long modelId); - - String getDictRootPath(); - - List searchDictLatestTaskList(DictLatestTaskReq filter, User user); -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java index 780af4cd3..fdf1e933f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java index aaed9a9d7..3b5e5f60f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.ViewSchema; @@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.DataInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.ViewInfo; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.chat.server.service.impl.SchemaService; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatKnowledgeTaskServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatKnowledgeTaskServiceImpl.java deleted file mode 100644 index daf6f58f5..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatKnowledgeTaskServiceImpl.java +++ /dev/null @@ -1,264 +0,0 @@ -package com.tencent.supersonic.chat.server.service.impl; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq; -import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; -import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp; -import com.tencent.supersonic.chat.core.config.DefaultMetric; -import com.tencent.supersonic.chat.core.config.Dim4Dict; -import com.tencent.supersonic.chat.core.knowledge.DictConfig; -import com.tencent.supersonic.chat.core.knowledge.DictUpdateMode; -import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand; -import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo; -import com.tencent.supersonic.chat.core.knowledge.FileHandler; -import com.tencent.supersonic.chat.core.utils.DictQueryHelper; -import com.tencent.supersonic.chat.server.listener.ApplicationStartedListener; -import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO; -import com.tencent.supersonic.chat.server.persistence.repository.DictRepository; -import com.tencent.supersonic.chat.server.service.KnowledgeTaskService; -import com.tencent.supersonic.chat.server.util.DictMetaHelper; -import com.tencent.supersonic.chat.server.util.DictTaskConverter; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.common.util.JsonUtil; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -@Slf4j -@Service -public class ChatKnowledgeTaskServiceImpl implements KnowledgeTaskService { - - private final DictMetaHelper metaUtils; - private final DictQueryHelper dictQueryHelper; - private final FileHandler fileHandler; - private final DictRepository dictRepository; - private final ApplicationStartedListener applicationStartedListener; - - @Value("${dict.flush.enable:true}") - private Boolean dictFlushEnable; - @Value("${dict.flush.daily.enable:true}") - private Boolean dictFlushDailyEnable; - @Value("${dict.file.type:txt}") - private String dictFileType; - private String dimValue = "DimValue_%d_%d"; - - public ChatKnowledgeTaskServiceImpl(DictMetaHelper metaUtils, - DictQueryHelper dictQueryHelper, - FileHandler fileHandler, - DictRepository dictRepository, - ApplicationStartedListener applicationStartedListener) { - this.metaUtils = metaUtils; - this.dictQueryHelper = dictQueryHelper; - this.fileHandler = fileHandler; - this.dictRepository = dictRepository; - this.applicationStartedListener = applicationStartedListener; - } - - @Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}") - public Boolean dailyDictTask() { - log.info("[dailyDictTask] start"); - if (!dictFlushDailyEnable) { - log.info("dictFlushDailyEnable is false, now finish dailyDictTask"); - } - DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand(); - dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL); - - User user = User.getFakeUser(); - addDictTask(dimValue2DictCommend, user); - log.info("[dailyDictTask] finish"); - return true; - } - - @Override - public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) { - if (!dictFlushEnable) { - return 0L; - } - - if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) { - return deleteDictTask(dimValue2DictCommend, user); - } - - DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user); - log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); - // todo check dimension can not be searched - - dictRepository.createDimValueDictTask(dictTaskDO); - runDictTask(dictTaskDO, user); - - return dictTaskDO.getId(); - } - - public Long runDictTask(DictTaskDO dictTaskDO, User user) { - if (Objects.isNull(dictTaskDO)) { - return -1L; - } - DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class); - try { - //1. construct internal dictionary requirements - List dimValueDOList = metaUtils.generateDimValueInfo(command); - Set dimIds = generateDimSet(dimValueDOList); - dictTaskDO.setDimIds(JsonUtil.toString(dimIds)); - dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO); - log.debug("dimValueDOList:{}", dimValueDOList); - //2. query dimension value information - for (DimValueDO dimValueDO : dimValueDOList) { - Long modelId = dimValueDO.getModelId(); - DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0); - for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) { - List data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user); - //3. local file changes - String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, - dim4Dict.getDimId()); - fileHandler.writeFile(data, fileName, false); - } - } - applicationStartedListener.updateKnowledgeDimValue(); - log.debug("updateDictTaskStatus to SUCCESS"); - dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO); - } catch (Exception e) { - log.warn("addDictInfo exception:", e); - dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO); - } - return 1L; - } - - private Set generateDimSet(List dimValueDOList) { - Set dimIds = new HashSet<>(); - if (!CollectionUtils.isEmpty(dimValueDOList)) { - dimValueDOList.stream().forEach(dimValueDO -> { - if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) { - dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId())); - } - }); - } - return dimIds; - } - - @Override - public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) { - if (!dictFlushEnable) { - return 0L; - } - if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals( - dimValue2DictCommand.getUpdateMode())) { - throw new RuntimeException("illegal parameter"); - } - - DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user); - log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO); - Set dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair()); - dictTaskDO.setDimIds(JsonUtil.toString(dimIds)); - dictRepository.createDimValueDictTask(dictTaskDO); - - Map> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair(); - if (CollectionUtils.isEmpty(modelAndDimPair)) { - return 0L; - } - for (Long modelId : modelAndDimPair.keySet()) { - if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) { - continue; - } - for (Long dimId : modelAndDimPair.get(modelId)) { - String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId); - fileHandler.deleteDictFile(fileName); - } - } - applicationStartedListener.updateKnowledgeDimValue(); - dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO); - applicationStartedListener.updateKnowledgeDimValue(); - - return 1L; - } - - private Set generateDimSetFromCommand(Map> modelAndDimPair) { - Set dimIds = new HashSet<>(); - if (!CollectionUtils.isEmpty(modelAndDimPair)) { - modelAndDimPair.forEach((k, v) -> dimIds.addAll(v)); - } - return dimIds; - } - - @Override - public String getDictRootPath() { - return fileHandler.getDictRootPath(); - } - - @Override - public List searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) { - DictTaskFilterReq filter = new DictTaskFilterReq(); - BeanUtils.copyProperties(latestFilter, filter); - List dimValueDictInfoList = searchDictTaskList(filter, user); - return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds()); - } - - private List extractLatestTask(List dimValueDictInfoList, List dimIds) { - List dictLatestTaskRespList = new ArrayList<>(); - Map dimAndTaskPair = new HashMap<>(50); - for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) { - //1. filter - if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) { - continue; - } - if (!CollectionUtils.isEmpty(dimIds)) { - Set tmp = dimValueDictInfo.getDimIds(); - tmp.retainAll(dimIds); - dimValueDictInfo.setDimIds(tmp); - if (tmp.size() <= 0) { - continue; - } - } - - // 2. extract - Set dimIdList = dimValueDictInfo.getDimIds(); - for (Long dimId : dimIdList) { - DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp(); - if (!dimAndTaskPair.containsKey(dimId)) { - BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp); - dictLatestTaskResp.setDimId(dimId); - } else { - DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId); - if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) { - BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp); - dictLatestTaskResp.setDimId(dimId); - } else { - dictLatestTaskResp = dictLatestTaskExist; - } - } - dimAndTaskPair.put(dimId, dictLatestTaskResp); - } - - } - - if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) { - dimAndTaskPair.values().stream() - .filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name())) - .forEach(v -> dictLatestTaskRespList.add(v)); - } - - - return dictLatestTaskRespList; - } - - @Override - public List searchDictTaskList(DictTaskFilterReq filter, User user) { - return dictRepository.searchDictTaskList(filter); - } - - @Override - public DictConfig getDictInfoByModelId(Long modelId) { - return dictRepository.getDictInfoByModelId(modelId); - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 0ab6160f5..5fb1c43ae 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/KnowledgeServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/KnowledgeServiceImpl.java deleted file mode 100644 index f2f563d7f..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/KnowledgeServiceImpl.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.tencent.supersonic.chat.server.service.impl; - -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import com.tencent.supersonic.chat.core.knowledge.SearchService; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; -import com.tencent.supersonic.chat.server.service.KnowledgeService; -import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.List; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -@Service -@Slf4j -public class KnowledgeServiceImpl implements KnowledgeService { - - public void updateSemanticKnowledge(List natures) { - - List prefixes = natures.stream() - .filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType())) - .collect(Collectors.toList()); - - for (DictWord nature : prefixes) { - HanlpHelper.addToCustomDictionary(nature); - } - - List suffixes = natures.stream() - .filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType())) - .collect(Collectors.toList()); - - SearchService.loadSuffix(suffixes); - } - - public void reloadAllData(List natures) { - // 1. reload custom knowledge - try { - HanlpHelper.reloadCustomDictionary(); - } catch (Exception e) { - log.error("reloadCustomDictionary error", e); - } - - // 2. update online knowledge - updateOnlineKnowledge(natures); - } - - public void updateOnlineKnowledge(List natures) { - try { - updateSemanticKnowledge(natures); - } catch (Exception e) { - log.error("updateSemanticKnowledge error", e); - } - } - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java index 7198e6d82..bef576c6a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java @@ -2,12 +2,12 @@ package com.tencent.supersonic.chat.server.service.impl; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; @@ -21,9 +21,9 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.corrector.SemanticCorrector; -import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.chat.core.knowledge.SearchService; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.SearchService; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.mapper.SchemaMapper; import com.tencent.supersonic.chat.core.parser.SemanticParser; import com.tencent.supersonic.chat.core.plugin.Plugin; @@ -33,8 +33,8 @@ import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; -import com.tencent.supersonic.chat.core.utils.NatureHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; import com.tencent.supersonic.chat.core.utils.SimilarQueryManager; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; @@ -671,7 +671,7 @@ public class QueryServiceImpl implements QueryService { } //search from prefixSearch List hanlpMapResultList = SearchService.prefixSearch(dimensionValueReq.getValue(), - 2000, dimensionValueReq.getAgentId(), detectModelIds); + 2000, detectModelIds); HanlpHelper.transLetterOriginal(hanlpMapResultList); return hanlpMapResultList.stream() .filter(o -> { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java index 6c9a1c105..fd9cf56e3 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.RecommendReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java index 4df5d17dc..19ba854d1 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java @@ -5,7 +5,7 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java index fdf56324c..d730f2043 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java @@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.server.service.impl; import com.github.benmanes.caffeine.cache.Cache; import com.google.common.collect.Lists; -import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; @@ -12,22 +11,24 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.SearchResult; import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat; import com.tencent.supersonic.chat.core.mapper.MapperHelper; import com.tencent.supersonic.chat.core.mapper.MatchText; import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType; import com.tencent.supersonic.chat.core.mapper.SearchMatchStrategy; import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; -import com.tencent.supersonic.chat.core.utils.NatureHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.chat.server.service.SearchService; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.server.service.KnowledgeService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -72,6 +73,9 @@ public class SearchServiceImpl implements SearchService { @Autowired private ConfigService configService; + @Autowired + private KnowledgeService knowledgeService; + @Override public List search(QueryReq queryReq) { // 1. check search enable @@ -90,7 +94,7 @@ public class SearchServiceImpl implements SearchService { final Map modelToName = semanticSchemaDb.getViewIdToName(); // 3.detect by segment - List originals = HanlpHelper.getTerms(queryText); + List originals = knowledgeService.getTerms(queryText); log.info("hanlp parse result: {}", originals); MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); Set detectModelIds = mapperHelper.getViewIds(queryReq.getModelId(), agentService.getAgent(agentId)); @@ -141,7 +145,7 @@ public class SearchServiceImpl implements SearchService { return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList()); } - private List getPossibleModels(QueryReq queryCtx, List originals, + private List getPossibleModels(QueryReq queryCtx, List originals, ViewInfoStat modelStat, Long webModelId) { if (Objects.nonNull(webModelId) && webModelId > 0) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java index b8c823175..fce843828 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat.server.service.impl; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import com.tencent.supersonic.chat.core.knowledge.builder.WordBuilderFactory; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java index 3da5ab77a..e47b84786 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java @@ -4,7 +4,7 @@ import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java index 1e6ef2faf..f3609398f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.util; import com.tencent.supersonic.chat.core.corrector.SemanticCorrector; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.mapper.SchemaMapper; import com.tencent.supersonic.chat.core.parser.SemanticParser; import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java deleted file mode 100644 index c19930e5b..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java +++ /dev/null @@ -1,247 +0,0 @@ -package com.tencent.supersonic.chat.server.util; - -import com.github.pagehelper.PageInfo; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig; -import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; -import com.tencent.supersonic.chat.core.config.DefaultMetric; -import com.tencent.supersonic.chat.core.config.Dim4Dict; -import com.tencent.supersonic.chat.core.knowledge.DictUpdateMode; -import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO; -import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; - -@Component -public class DictMetaHelper { - - @Autowired - private ConfigService configService; - @Value("${model.internal.metric.suffix:internal_cnt}") - private String internalMetricNameSuffix; - @Value("${model.internal.day.number:2}") - private Integer internalMetricDays; - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - - public List generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) { - List dimValueDOList = new ArrayList<>(); - DictUpdateMode updateMode = dimValue2DictCommend.getUpdateMode(); - Set modelIds = new HashSet<>(); - switch (updateMode) { - case OFFLINE_MODEL: - modelIds.addAll(dimValue2DictCommend.getModelIds()); - dimValueDOList = generateDimValueInfoByModel(modelIds); - break; - case OFFLINE_FULL: - List modelSchemaDescList = semanticInterpreter.getViewSchema(); - if (CollectionUtils.isEmpty(modelSchemaDescList)) { - break; - } - - Map modelIdAndDescPair = modelSchemaDescList.stream() - .collect(Collectors.toMap(a -> a.getView().getId(), schema -> schema, (k1, k2) -> k1)); - if (!CollectionUtils.isEmpty(modelIdAndDescPair)) { - modelIds.addAll(modelIdAndDescPair.keySet()); - dimValueDOList = generateDimValueInfoByModel(modelIds); - break; - } - break; - case REALTIME_ADD: - dimValueDOList = generateDimValueInfoByModelAndDim(dimValue2DictCommend.getModelAndDimPair()); - break; - case NOT_SUPPORT: - throw new RuntimeException("illegal parameter for updateMode"); - default: - break; - } - - return dimValueDOList; - } - - private List generateDimValueInfoByModelAndDim(Map> modelAndDimMap) { - List dimValueDOList = new ArrayList<>(); - if (CollectionUtils.isEmpty(modelAndDimMap)) { - return dimValueDOList; - } - - List modelSchemaDescList = semanticInterpreter.getViewSchema(); - if (CollectionUtils.isEmpty(modelSchemaDescList)) { - return dimValueDOList; - } - Map modelIdAndDescPair = modelSchemaDescList.stream() - .collect(Collectors.toMap(a -> a.getView().getId(), a -> a, (k1, k2) -> k1)); - - for (Long modelId : modelAndDimMap.keySet()) { - if (!modelIdAndDescPair.containsKey(modelId)) { - continue; - } - Map dimIdAndDescPairAll; - dimIdAndDescPairAll = modelIdAndDescPair.get(modelId).getDimensions().stream() - .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1)); - List dimIdReq = modelAndDimMap.get(modelId); - Map dimIdAndDescPairReq = new HashMap<>(); - for (Long dimId : dimIdReq) { - if (dimIdAndDescPairAll.containsKey(dimId)) { - dimIdAndDescPairReq.put(dimId, dimIdAndDescPairAll.get(dimId)); - } - } - fillDimValueDOList(dimValueDOList, modelId, dimIdAndDescPairReq); - } - - return dimValueDOList; - } - - private List generateDimValueInfoByModel(Set modelIds) { - List dimValueDOList = new ArrayList<>(); - List modelSchemaDescList = semanticInterpreter.getViewSchema(new ArrayList<>(modelIds)); - if (CollectionUtils.isEmpty(modelSchemaDescList)) { - return dimValueDOList; - } - - modelSchemaDescList.forEach(modelSchemaDesc -> { - Map dimIdAndDescPair = modelSchemaDesc.getDimensions().stream() - .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1)); - fillDimValueDOList(dimValueDOList, modelSchemaDesc.getView().getId(), dimIdAndDescPair); - - }); - - return dimValueDOList; - } - - private void fillDimValueDOList(List dimValueDOList, Long modelId, - Map dimIdAndDescPair) { - ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(modelId); - if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatAggRichConfig())) { - - ChatDefaultRichConfigResp chatDefaultConfig = - chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig(); - - List knowledgeAggInfo = - chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos(); - - List knowledgeDetailInfo = - chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos(); - - fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId); - fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId); - - - } - } - - private void fillKnowledgeDimValue(List knowledgeInfos, - ChatDefaultRichConfigResp chatDefaultConfig, - List dimValueDOList, - Map dimIdAndDescPair, Long modelId) { - Map dimIdAndRespPair = queryDimensionRespByModelId( - new ArrayList<>(Arrays.asList(modelId))); - if (!CollectionUtils.isEmpty(knowledgeInfos)) { - List dimensions = new ArrayList<>(); - List defaultMetricDescList = new ArrayList<>(); - knowledgeInfos.stream() - .filter(knowledgeInfo -> knowledgeInfo.getSearchEnable() - && !CollectionUtils.isEmpty(dimIdAndDescPair) - && dimIdAndDescPair.containsKey(knowledgeInfo.getItemId())) - .forEach(knowledgeInfo -> { - SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId()); - - Long dimId = dimensionDesc.getId(); - String internalMetricName = ""; - if (Objects.nonNull(dimId)) { - String datasourceBizName = queryDataSourceByDimId(dimId); - internalMetricName = datasourceBizName + UNDERLINE + internalMetricNameSuffix; - } - - if (Objects.isNull(chatDefaultConfig)) { - defaultMetricDescList.add(new DefaultMetric(internalMetricName, - internalMetricDays, DAY)); - } else { - String metric = internalMetricName; - if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) { - metric = chatDefaultConfig.getMetrics().get(0).getBizName(); - } - defaultMetricDescList.add(new DefaultMetric(metric, - chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod())); - - } - - String bizName = dimensionDesc.getBizName(); - Dim4Dict dim4Dict = new Dim4Dict(); - dim4Dict.setDimId(knowledgeInfo.getItemId()); - dim4Dict.setBizName(bizName); - if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) { - KnowledgeAdvancedConfig knowledgeAdvancedConfig - = knowledgeInfo.getKnowledgeAdvancedConfig(); - BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict); - - if (Objects.nonNull(dimIdAndRespPair) - && dimIdAndRespPair.containsKey(dim4Dict.getDimId())) { - String datasourceFilterSql = dimIdAndRespPair.get( - dim4Dict.getDimId()).getModelFilterSql(); - if (StringUtils.isNotEmpty(datasourceFilterSql)) { - dim4Dict.getRuleList().add(datasourceFilterSql); - } - - } - } - dimensions.add(dim4Dict); - - }); - - if (!CollectionUtils.isEmpty(dimensions)) { - DimValueDO dimValueDO = new DimValueDO() - .setModelId(modelId) - .setDefaultMetricIds(defaultMetricDescList) - .setDimensions(dimensions); - dimValueDOList.add(dimValueDO); - } - } - } - - private Map queryDimensionRespByModelId(List modelIds) { - Map dimIdAndRespPair = new HashMap<>(); - PageDimensionReq pageDimensionCmd = new PageDimensionReq(); - pageDimensionCmd.setModelIds(modelIds); - PageInfo dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd); - if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) { - List dimList = dimensionPage.getList(); - dimIdAndRespPair = dimList.stream().collect(Collectors.toMap(DimensionResp::getId, v -> v, (v1, v2) -> v2)); - } - return dimIdAndRespPair; - } - - private String queryDataSourceByDimId(Long id) { - PageDimensionReq pageDimensionCmd = new PageDimensionReq(); - pageDimensionCmd.setId(id.toString()); - PageInfo dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd); - if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) { - List list = dimensionPage.getList(); - return list.get(0).getModelBizName(); - } - return ""; - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictTaskConverter.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictTaskConverter.java deleted file mode 100644 index f1f6858f9..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictTaskConverter.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.tencent.supersonic.chat.server.util; - -import com.google.common.base.Strings; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.knowledge.DictConfig; -import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand; -import com.tencent.supersonic.chat.core.knowledge.DimValueInfo; -import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO; -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.common.util.JsonUtil; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.List; - -public class DictTaskConverter { - - private static String dateTimeFormatter = "yyyyMMddHHmmss"; - - public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) { - DictTaskDO taskPO = new DictTaskDO(); - Date createAt = new Date(); - String date = DateTimeFormatter.ofPattern(dateTimeFormatter) - .format(createAt.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime()); - String creator = Strings.isNullOrEmpty(user.getName()) ? "" : user.getName(); - String updateMode = dimValue2DictCommend.getUpdateMode().getValue(); - String name = String.format("DimValue_dic_%s_%s_%s", updateMode, creator, date); - taskPO.setName(name); - - taskPO.setCreatedAt(createAt); - taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend)); - taskPO.setStatus(TaskStatusEnum.PENDING.getCode()); - taskPO.setCreatedBy(creator); - - return taskPO; - } - - public static DictConfig dictConfPO2Config(DictConfDO dictConfDO) { - DictConfig dictConfig = new DictConfig(); - dictConfig.setModelId(dictConfDO.getModelId()); - List dimValueInfos = JsonUtil.toList(dictConfDO.getDimValueInfos(), DimValueInfo.class); - dictConfig.setDimValueInfoList(dimValueInfos); - return dictConfig; - } -} diff --git a/chat/server/src/main/resources/mapper/DictConfMapper.xml b/chat/server/src/main/resources/mapper/DictConfMapper.xml deleted file mode 100644 index 5ec8d968a..000000000 --- a/chat/server/src/main/resources/mapper/DictConfMapper.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - insert into s2_dictionary - (`domain_id`, dim_value_infos, created_at, updated_at, created_by, updated_by) - values - (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy}) - - - - insert into s2_dictionary - (`model_id`, dim_value_infos, created_at, updated_at, created_by, updated_by) - values - (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy}) - on duplicate key update - dim_value_infos = #{dimValueInfos}, - updated_at = #{updatedAt}, - updated_by = #{updatedBy} - - - - update s2_dictionary - set dim_value_infos = #{dimValueInfos}, - updated_at = #{updatedAt}, - updated_by = #{updatedBy} - where model_id = #{modelId} - and status = 0 - - - - - - \ No newline at end of file diff --git a/chat/server/src/main/resources/mapper/DictTaskMapper.xml b/chat/server/src/main/resources/mapper/DictTaskMapper.xml deleted file mode 100644 index 807357dd4..000000000 --- a/chat/server/src/main/resources/mapper/DictTaskMapper.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - insert into s2_dictionary_task - (`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms) - values - (#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs}) - - - - update s2_dictionary_task - - - description = #{description}, - - - status = #{status}, - - - dimension_ids = #{dimIds}, - - - progress = #{progress}, - - - elapsed_ms = #{elapsedMs}, - - - - where id = #{id} - - - - - \ No newline at end of file diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java new file mode 100644 index 000000000..e69de29bb diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java index 0a23b439a..1ab50b8f3 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.test.context; import com.google.gson.Gson; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; diff --git a/common/pom.xml b/common/pom.xml index 97d3ae7d4..887db13b4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -190,6 +190,11 @@ dev.langchain4j langchain4j-embeddings + + com.hankcs + hanlp + ${hanlp.version} + diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java similarity index 64% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java rename to common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java index be6bb2b1c..fc2850513 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java +++ b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java @@ -1,10 +1,10 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.hankcs.hanlp; -import com.tencent.supersonic.chat.core.utils.NatureHelper; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import java.util.ArrayList; @@ -13,16 +13,13 @@ import java.util.Objects; import java.util.Set; @Data -@Service +@Slf4j public class LoadRemoveService { - @Value("${mapper.remove.agentId:}") - private Integer mapperRemoveAgentId; - @Value("${mapper.remove.nature.prefix:}") private String mapperRemoveNaturePrefix; - public List removeNatures(List value, Integer agentId, Set detectModelIds) { + public List removeNatures(List value, Set detectModelIds) { if (CollectionUtils.isEmpty(value)) { return value; } @@ -32,16 +29,14 @@ public class LoadRemoveService { if (Objects.isNull(nature)) { return false; } - Long modelId = NatureHelper.getViewId(nature); + Long modelId = getViewId(nature); if (Objects.nonNull(modelId)) { return !detectModelIds.contains(modelId); } return false; }); } - if (Objects.nonNull(mapperRemoveAgentId) - && mapperRemoveAgentId.equals(agentId) - && StringUtils.isNotBlank(mapperRemoveNaturePrefix)) { + if (StringUtils.isNotBlank(mapperRemoveNaturePrefix)) { resultList.removeIf(nature -> { if (Objects.isNull(nature)) { return false; @@ -52,4 +47,17 @@ public class LoadRemoveService { return resultList; } + public Long getViewId(String nature) { + try { + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length <= 1) { + return null; + } + return Long.valueOf(split[1]); + } catch (NumberFormatException e) { + log.error("", e); + } + return null; + } + } diff --git a/chat/core/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java b/common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java similarity index 92% rename from chat/core/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java rename to common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java index e744ecc4b..0f50c76ba 100644 --- a/chat/core/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java +++ b/common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java @@ -1,8 +1,7 @@ package com.hankcs.hanlp.collection.trie.bintrie; +import com.hankcs.hanlp.LoadRemoveService; import com.hankcs.hanlp.corpus.io.ByteArray; -import com.tencent.supersonic.chat.core.knowledge.LoadRemoveService; -import com.tencent.supersonic.common.util.ContextUtils; import java.io.DataOutputStream; import java.io.IOException; import java.io.ObjectInput; @@ -287,11 +286,10 @@ public abstract class BaseNode implements Comparable { + '}'; } - public void walkNode(Set> entrySet, Integer agentId, Set detectModelIds) { + public void walkNode(Set> entrySet, Set detectModelIds) { if (status == Status.WORD_MIDDLE_2 || status == Status.WORD_END_3) { - LoadRemoveService loadRemoveService = ContextUtils.getBean(LoadRemoveService.class); - logger.debug("agentId:{},detectModelIds:{},before:{}", agentId, detectModelIds, value.toString()); - List natures = loadRemoveService.removeNatures((List) value, agentId, detectModelIds); + logger.debug("detectModelIds:{},before:{}", detectModelIds, value.toString()); + List natures = new LoadRemoveService().removeNatures((List) value, detectModelIds); String name = this.prefix != null ? this.prefix + c : "" + c; logger.debug("name:{},after:{},natures:{}", name, (List) value, natures); entrySet.add(new TrieEntry(name, (V) natures)); @@ -304,8 +302,7 @@ public abstract class BaseNode implements Comparable { * @param entrySet * @param limit */ - public void walkLimit(StringBuilder sb, Set> entrySet, int limit, Integer agentId, - Set detectModelIds) { + public void walkLimit(StringBuilder sb, Set> entrySet, int limit, Set detectModelIds) { Queue queue = new ArrayDeque<>(); this.prefix = sb.toString(); queue.add(this); @@ -317,7 +314,7 @@ public abstract class BaseNode implements Comparable { if (root == null) { continue; } - root.walkNode(entrySet, agentId, detectModelIds); + root.walkNode(entrySet, detectModelIds); if (root.child == null) { continue; } diff --git a/chat/core/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java b/common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java similarity index 93% rename from chat/core/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java rename to common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java index e97c9f903..dfef774d6 100644 --- a/chat/core/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java +++ b/common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java @@ -1,8 +1,6 @@ package com.hankcs.hanlp.dictionary; -import static com.hankcs.hanlp.utility.Predefine.logger; - import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.collection.trie.DoubleArrayTrie; import com.hankcs.hanlp.corpus.io.ByteArray; @@ -36,7 +34,8 @@ public class CoreDictionary { if (!load(PATH)) { throw new IllegalArgumentException("核心词典" + PATH + "加载失败"); } else { - logger.info(PATH + "加载成功，" + trie.size() + "个词条，耗时" + (System.currentTimeMillis() - start) + "ms"); + Predefine.logger.info(PATH + "加载成功，" + trie.size() + "个词条，耗时" + + (System.currentTimeMillis() - start) + "ms"); } } @@ -50,7 +49,7 @@ public class CoreDictionary { public static final int NX_WORD_ID = getWordID(Predefine.TAG_PROPER); private static boolean load(String path) { - logger.info("核心词典开始加载:" + path); + Predefine.logger.info("核心词典开始加载:" + path); if (loadDat(path)) { return true; } @@ -73,12 +72,12 @@ public class CoreDictionary { map.put(param[0], attribute); totalFrequency += attribute.totalFrequency; } - logger.info( + Predefine.logger.info( "核心词典读入词条" + map.size() + " 全部频次" + totalFrequency + "，耗时" + (System.currentTimeMillis() - start) + "ms"); br.close(); trie.build(map); - logger.info("核心词典加载成功:" + trie.size() + "个词条，下面将写入缓存……"); + Predefine.logger.info("核心词典加载成功:" + trie.size() + "个词条，下面将写入缓存……"); try { DataOutputStream out = new DataOutputStream( new BufferedOutputStream(IOUtil.newOutputStream(path + Predefine.BIN_EXT))); @@ -97,14 +96,14 @@ public class CoreDictionary { Predefine.setTotalFrequency(totalFrequency); out.close(); } catch (Exception e) { - logger.warning("保存失败" + e); + Predefine.logger.warning("保存失败" + e); return false; } } catch (FileNotFoundException e) { - logger.warning("核心词典" + path + "不存在！" + e); + Predefine.logger.warning("核心词典" + path + "不存在！" + e); return false; } catch (IOException e) { - logger.warning("核心词典" + path + "读取错误！" + e); + Predefine.logger.warning("核心词典" + path + "读取错误！" + e); return false; } @@ -150,7 +149,7 @@ public class CoreDictionary { } Predefine.setTotalFrequency(totalFrequency); } catch (Exception e) { - logger.warning("读取失败，问题发生在" + e); + Predefine.logger.warning("读取失败，问题发生在" + e); return false; } return true; @@ -265,7 +264,8 @@ public class CoreDictionary { } return attribute; } catch (Exception e) { - logger.warning("使用字符串" + natureWithFrequency + "创建词条属性失败！" + TextUtility.exceptionToString(e)); + Predefine.logger.warning("使用字符串" + natureWithFrequency + "创建词条属性失败！" + + TextUtility.exceptionToString(e)); return null; } } diff --git a/chat/core/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java b/common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java similarity index 98% rename from chat/core/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java rename to common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java index 49571493a..41d356e99 100644 --- a/chat/core/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java +++ b/common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java @@ -6,14 +6,15 @@ import com.hankcs.hanlp.collection.AhoCorasick.AhoCorasickDoubleArrayTrie; import com.hankcs.hanlp.collection.trie.DoubleArrayTrie; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; +import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.dictionary.CoreDictionaryTransformMatrixDictionary; import com.hankcs.hanlp.dictionary.other.CharType; import com.hankcs.hanlp.seg.NShort.Path.AtomNode; import com.hankcs.hanlp.seg.common.Graph; -import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.seg.common.Vertex; import com.hankcs.hanlp.seg.common.WordNet; import com.hankcs.hanlp.utility.TextUtility; + import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; @@ -135,7 +136,7 @@ public abstract class WordBasedSegment extends Segment { } protected static List convert(List vertexList) { - return convert(vertexList, false); + return Segment.convert(vertexList, false); } protected static Graph generateBiGraph(WordNet wordNet) { @@ -269,7 +270,7 @@ public abstract class WordBasedSegment extends Segment { ++j) { } - wordNetStorage.add(i, quickAtomSegment(charArray, i - 1, j - 1)); + wordNetStorage.add(i, Segment.quickAtomSegment(charArray, i - 1, j - 1)); i = j; } else { i += ((Vertex) vertexes[i].getLast()).realWord.length(); @@ -289,7 +290,7 @@ public abstract class WordBasedSegment extends Segment { for (int i = 0; i < length; ++i) { Vertex vertex = (Vertex) listIterator.next(); - Term termMain = convert(vertex); + Term termMain = Segment.convert(vertex); //termList.add(termMain); addTerms(termList, vertex, line - 1); termMain.offset = line - 1; diff --git a/chat/core/src/main/java/com/hankcs/hanlp/seg/common/Term.java b/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java similarity index 80% rename from chat/core/src/main/java/com/hankcs/hanlp/seg/common/Term.java rename to common/src/main/java/com/hankcs/hanlp/seg/common/Term.java index 5cf122ebb..78ec273e2 100644 --- a/chat/core/src/main/java/com/hankcs/hanlp/seg/common/Term.java +++ b/common/src/main/java/com/hankcs/hanlp/seg/common/Term.java @@ -1,12 +1,14 @@ package com.hankcs.hanlp.seg.common; import com.hankcs.hanlp.corpus.tag.Nature; -import com.hankcs.hanlp.dictionary.CoreDictionary; -import com.hankcs.hanlp.dictionary.CustomDictionary; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; +//import com.hankcs.hanlp.dictionary.CoreDictionary; +//import com.hankcs.hanlp.dictionary.CustomDictionary; +//import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import lombok.Data; import lombok.ToString; +//import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; + @Data @ToString public class Term { @@ -43,8 +45,10 @@ public class Term { if (frequency > 0) { return frequency; } + // todo opt + /* String wordOri = word.toLowerCase(); - CoreDictionary.Attribute attribute = HanlpHelper.getDynamicCustomDictionary().get(wordOri); + CoreDictionary.Attribute attribute = getDynamicCustomDictionary().get(wordOri); if (attribute == null) { attribute = CoreDictionary.get(wordOri); if (attribute == null) { @@ -55,6 +59,8 @@ public class Term { return attribute.getNatureFrequency(nature); } return attribute == null ? 0 : attribute.totalFrequency; + */ + return 0; } public boolean equals(Object obj) { @@ -66,4 +72,5 @@ public class Term { } return super.equals(obj); } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java new file mode 100644 index 000000000..c108012cf --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +/** + * 导入字典的可选配置 + */ + +@Data +public class ItemValueConfig { + + private Long metricId; + private List blackList = new ArrayList<>(); + private List whiteList = new ArrayList<>(); + private List ruleList = new ArrayList<>(); + private Long limit; +} \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/RelatedSchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java similarity index 84% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/RelatedSchemaElement.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java index 91c1671c0..e8fd6d014 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/RelatedSchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/RelatedSchemaElement.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; import lombok.AllArgsConstructor; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java similarity index 96% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java index f7105b86b..22e50df9e 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; import com.google.common.base.Objects; import com.google.common.collect.Lists; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java similarity index 70% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java index ab254c538..c64f8d4b8 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; public enum SchemaElementType { VIEW, diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaValueMap.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java similarity index 88% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaValueMap.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java index e1dd88f5b..bea54fac6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaValueMap.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaValueMap.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; import java.util.ArrayList; import java.util.List; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java new file mode 100644 index 000000000..089bbba1a --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import lombok.Builder; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +@Data +@Builder +public class DictItemFilter { + private Long id; + private TypeEnums type; + private Long itemId; + @NotNull + private StatusEnum status; +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java new file mode 100644 index 000000000..22ad7cc5a --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemReq.java @@ -0,0 +1,27 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +@Data +public class DictItemReq { + + private Long id; + @NotNull + private TypeEnums type; + @NotNull + private Long itemId; + private ItemValueConfig config; + + /** + * ONLINE - 正常更新 + * OFFLINE - 停止更新,但字典文件不删除 + * DELETED - 停止更新,且删除字典文件 + */ + @NotNull + private StatusEnum status; +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java new file mode 100644 index 000000000..b4726af44 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictSingleTaskReq.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import lombok.Builder; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +@Data +@Builder +public class DictSingleTaskReq { + @NotNull + private TypeEnums type; + @NotNull + private Long itemId; +} \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java similarity index 82% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java index 77b50a8c6..82b361d1c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionValueReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.request; +package com.tencent.supersonic.headless.api.pojo.request; import javax.validation.constraints.NotNull; import lombok.Data; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java new file mode 100644 index 000000000..ab7c727b2 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java @@ -0,0 +1,41 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; + +@Data +public class DictItemResp { + private Long id; + + private Long modelId; + + private String bizName; + + @NotNull + private TypeEnums type; + @NotNull + private Long itemId; + private ItemValueConfig config; + + /** + * ONLINE - 正常更新 + * OFFLINE - 停止更新,但字典文件不删除 + * DELETED - 停止更新,且删除字典文件 + */ + @NotNull + private StatusEnum status; + + public String getNature() { + return UNDERLINE + modelId + UNDERLINE + itemId; + } + + public String fetchDictFileName() { + return String.format("dic_value_%d_%s_%s", modelId, type.name(), itemId); + } +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java new file mode 100644 index 000000000..cee376e44 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictTaskResp.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.headless.api.pojo.response; + + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class DictTaskResp extends DictItemResp { + + private String name; + private String description; + private String taskStatus; + private Date createdAt; + private String createdBy; + private Long elapsedMs; + +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java new file mode 100644 index 000000000..80e6be4f6 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/S2Term.java @@ -0,0 +1,50 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.hankcs.hanlp.corpus.tag.Nature; +import com.hankcs.hanlp.seg.common.Term; +import lombok.Data; + +@Data +public class S2Term { + + public String word; + + public Nature nature; + public int offset; + public int frequency = 0; + + public S2Term() { + } + + public S2Term(String word, Nature nature) { + this.word = word; + this.nature = nature; + } + + public S2Term(String word, Nature nature, int offset) { + this.word = word; + this.nature = nature; + this.offset = offset; + } + + public S2Term(String word, Nature nature, int offset, int frequency) { + this.word = word; + this.nature = nature; + this.offset = offset; + this.frequency = frequency; + } + + public int length() { + return this.word.length(); + } + + public boolean equals(Object obj) { + if (obj instanceof Term) { + Term term = (Term) obj; + if (this.nature == term.nature && this.word.equals(term.word)) { + return true; + } + } + return super.equals(obj); + } +} \ No newline at end of file diff --git a/headless/core/pom.xml b/headless/core/pom.xml index 4d49a1ee8..3a9a88924 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -75,6 +75,44 @@ + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + * + + + javax.servlet + servlet-api + + + + + + + + + + org.apache.commons + commons-compress + ${commons.compress.version} + compile + diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/FileHandler.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandler.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/FileHandler.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandler.java index e2482268b..3242e0cf2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/FileHandler.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandler.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.file; import java.util.List; @@ -53,4 +53,5 @@ public interface FileHandler { * @param fileName */ void deleteFile(String fileName); + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LocalFileHandler.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandlerImpl.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LocalFileHandler.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandlerImpl.java index 6a6c07ed4..c22573671 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LocalFileHandler.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandlerImpl.java @@ -1,27 +1,26 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.file; - -import com.tencent.supersonic.chat.core.config.LocalFileConfig; -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.charset.StandardCharsets; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; @Slf4j @Component -public class LocalFileHandler implements FileHandler { +public class FileHandlerImpl implements FileHandler { private final LocalFileConfig localFileConfig; - public LocalFileHandler(LocalFileConfig localFileConfig) { + public FileHandlerImpl(LocalFileConfig localFileConfig) { this.localFileConfig = localFileConfig; } @@ -81,6 +80,10 @@ public class LocalFileHandler implements FileHandler { @Override public void writeFile(List lines, String fileName, Boolean append) { + if (CollectionUtils.isEmpty(lines)) { + log.info("lines is empty"); + return; + } String dictDirectoryLatest = localFileConfig.getDictDirectoryLatest(); if (!existPath(dictDirectoryLatest)) { createDir(dictDirectoryLatest); @@ -124,4 +127,4 @@ public class LocalFileHandler implements FileHandler { } return Files.newBufferedWriter(Paths.get(filePath), StandardCharsets.UTF_8); } -} +} \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/LocalFileConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/LocalFileConfig.java new file mode 100644 index 000000000..6e2c8b4ba --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/file/LocalFileConfig.java @@ -0,0 +1,33 @@ +package com.tencent.supersonic.headless.core.file; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration +@Slf4j +public class LocalFileConfig { + + + @Value("${dict.directory.latest:/data/dictionary/custom}") + private String dictDirectoryLatest; + + @Value("${dict.directory.backup:./data/dictionary/backup}") + private String dictDirectoryBackup; + + public String getDictDirectoryLatest() { + return getResourceDir() + dictDirectoryLatest; + } + + public String getDictDirectoryBackup() { + return getResourceDir() + dictDirectoryBackup; + } + + private String getResourceDir() { + //return hanlpPropertiesPath = HanlpHelper.getHanlpPropertiesPath(); + return ClassLoader.getSystemClassLoader().getResource("").getPath(); + } + +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DatabaseMapResult.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DatabaseMapResult.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DatabaseMapResult.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DatabaseMapResult.java index b58c36fd0..10871d3d7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DatabaseMapResult.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DatabaseMapResult.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import com.google.common.base.Objects; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.Data; import lombok.ToString; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictUpdateMode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictUpdateMode.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictUpdateMode.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictUpdateMode.java index eba4b6727..b70b987c7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictUpdateMode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictUpdateMode.java @@ -1,5 +1,5 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; public enum DictUpdateMode { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictWord.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictWord.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictWord.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictWord.java index 9241991eb..810749c85 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictWord.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictWord.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import java.util.Objects; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictionaryAttributeUtil.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictionaryAttributeUtil.java similarity index 96% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictionaryAttributeUtil.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictionaryAttributeUtil.java index d3d4f0900..a05bfe7bf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictionaryAttributeUtil.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictionaryAttributeUtil.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge.dictionary; +package com.tencent.supersonic.headless.core.knowledge; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/EmbeddingResult.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/EmbeddingResult.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/EmbeddingResult.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/EmbeddingResult.java index c664b5a31..231e37c38 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/EmbeddingResult.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/EmbeddingResult.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import com.google.common.base.Objects; import java.util.Map; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/HadoopFileIOAdapter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HadoopFileIOAdapter.java similarity index 94% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/HadoopFileIOAdapter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HadoopFileIOAdapter.java index b4b7f3ba8..e215ca782 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/HadoopFileIOAdapter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HadoopFileIOAdapter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import com.hankcs.hanlp.corpus.io.IIOAdapter; import java.io.IOException; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/HanlpMapResult.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HanlpMapResult.java similarity index 94% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/HanlpMapResult.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HanlpMapResult.java index 8a363c06a..ba198ec00 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/HanlpMapResult.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HanlpMapResult.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import com.google.common.base.Objects; import java.util.List; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/MapResult.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MapResult.java similarity index 78% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/MapResult.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MapResult.java index 2287933e8..23804123a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/MapResult.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MapResult.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import java.io.Serializable; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/MultiCustomDictionary.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MultiCustomDictionary.java similarity index 98% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/MultiCustomDictionary.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MultiCustomDictionary.java index cda2fdde1..194da75a7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/MultiCustomDictionary.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MultiCustomDictionary.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import static com.hankcs.hanlp.utility.Predefine.logger; @@ -15,8 +15,8 @@ import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.utility.LexiconUtility; import com.hankcs.hanlp.utility.Predefine; import com.hankcs.hanlp.utility.TextUtility; -import com.tencent.supersonic.chat.core.knowledge.dictionary.DictionaryAttributeUtil; -import com.tencent.supersonic.chat.core.utils.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; + import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.DataOutputStream; @@ -166,7 +166,8 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { * @return */ public static boolean loadMainDictionary(String mainPath, String[] path, - DoubleArrayTrie dat, boolean isCache, boolean addToSuggestTrie) { + DoubleArrayTrie dat, boolean isCache, + boolean addToSuggestTrie) { Predefine.logger.info("自定义词典开始加载:" + mainPath); if (loadDat(mainPath, dat)) { return true; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/SearchService.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java similarity index 87% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/SearchService.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java index e6e95ebb3..61809f67e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/SearchService.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java @@ -1,12 +1,10 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import com.hankcs.hanlp.collection.trie.bintrie.BaseNode; import com.hankcs.hanlp.collection.trie.bintrie.BinTrie; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.chat.core.knowledge.dictionary.DictionaryAttributeUtil; import com.tencent.supersonic.common.pojo.enums.DictWordType; import java.util.ArrayList; import java.util.Arrays; @@ -18,6 +16,8 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; + +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -39,13 +39,13 @@ public class SearchService { * @param key * @return */ - public static List prefixSearch(String key, int limit, Integer agentId, Set detectModelIds) { - return prefixSearch(key, limit, agentId, trie, detectModelIds); + public static List prefixSearch(String key, int limit, Set detectModelIds) { + return prefixSearch(key, limit, trie, detectModelIds); } - public static List prefixSearch(String key, int limit, Integer agentId, - BinTrie> binTrie, Set detectModelIds) { - Set>> result = prefixSearchLimit(key, limit, binTrie, agentId, detectModelIds); + public static List prefixSearch(String key, int limit, BinTrie> binTrie, + Set detectModelIds) { + Set>> result = prefixSearchLimit(key, limit, binTrie, detectModelIds); return result.stream().map( entry -> { String name = entry.getKey().replace("#", " "); @@ -61,14 +61,14 @@ public class SearchService { * @param key * @return */ - public static List suffixSearch(String key, int limit, Integer agentId, Set detectModelIds) { + public static List suffixSearch(String key, int limit, Set detectModelIds) { String reverseDetectSegment = StringUtils.reverse(key); - return suffixSearch(reverseDetectSegment, limit, agentId, suffixTrie, detectModelIds); + return suffixSearch(reverseDetectSegment, limit, suffixTrie, detectModelIds); } - public static List suffixSearch(String key, int limit, Integer agentId, - BinTrie> binTrie, Set detectModelIds) { - Set>> result = prefixSearchLimit(key, limit, binTrie, agentId, detectModelIds); + public static List suffixSearch(String key, int limit, BinTrie> binTrie, + Set detectModelIds) { + Set>> result = prefixSearchLimit(key, limit, binTrie, detectModelIds); return result.stream().map( entry -> { String name = entry.getKey().replace("#", " "); @@ -84,7 +84,7 @@ public class SearchService { } private static Set>> prefixSearchLimit(String key, int limit, - BinTrie> binTrie, Integer agentId, Set detectModelIds) { + BinTrie> binTrie, Set detectModelIds) { key = key.toLowerCase(); Set>> entrySet = new TreeSet>>(); @@ -104,7 +104,7 @@ public class SearchService { if (branch == null) { return entrySet; } - branch.walkLimit(sb, entrySet, limit, agentId, detectModelIds); + branch.walkLimit(sb, entrySet, limit, detectModelIds); return entrySet; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java index d420e1995..00ba8d2af 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge; +package com.tencent.supersonic.headless.core.knowledge; import lombok.Builder; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/BaseWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/BaseWordBuilder.java similarity index 83% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/BaseWordBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/BaseWordBuilder.java index 590c04b9c..4cbb2ff8b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/BaseWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/BaseWordBuilder.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.core.knowledge.DictWord; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java index e000aef00..96247a048 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.core.knowledge.DictWord; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java similarity index 86% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java index 1f14d7971..8ce190e2c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.core.knowledge.DictWord; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java index 6fe0cecd6..c54fca038 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.core.knowledge.DictWord; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java similarity index 86% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java index d388dd3a3..7f371a139 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.core.knowledge.DictWord; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java similarity index 86% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java index 95f5e8aea..73f1bbbc0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.core.knowledge.DictWord; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/WordBuilderFactory.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/WordBuilderFactory.java index f13e2417c..678e8aacf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/WordBuilderFactory.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.knowledge.builder; +package com.tencent.supersonic.headless.core.knowledge.builder; import com.tencent.supersonic.common.pojo.enums.DictWordType; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/FileHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/FileHelper.java similarity index 97% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/FileHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/FileHelper.java index 2fb3121e5..5da203f3f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/FileHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/FileHelper.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.knowledge.helper; import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/HanlpHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/HanlpHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java index 136e3da09..2883babce 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/HanlpHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.knowledge.helper; import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; @@ -7,13 +7,14 @@ import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.seg.Segment; -import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.core.knowledge.DictWord; -import com.tencent.supersonic.chat.core.knowledge.HadoopFileIOAdapter; -import com.tencent.supersonic.chat.core.knowledge.MapResult; -import com.tencent.supersonic.chat.core.knowledge.MultiCustomDictionary; -import com.tencent.supersonic.chat.core.knowledge.SearchService; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.HadoopFileIOAdapter; +import com.tencent.supersonic.headless.core.knowledge.MapResult; +import com.tencent.supersonic.headless.core.knowledge.MultiCustomDictionary; +import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.common.pojo.enums.DictWordType; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -21,8 +22,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; + +import com.hankcs.hanlp.seg.common.Term; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; import org.springframework.util.ResourceUtils; @@ -89,11 +93,11 @@ public class HanlpHelper { } if (HanLP.Config.IOAdapter instanceof HadoopFileIOAdapter) { // 1.delete hdfs file - HdfsFileHelper.deleteCacheFile(CustomDictionaryPath); + HdfsFileHelper.deleteCacheFile(HanLP.Config.CustomDictionaryPath); // 2.query txt files，update CustomDictionaryPath HdfsFileHelper.resetCustomPath(getDynamicCustomDictionary()); } else { - FileHelper.deleteCacheFile(CustomDictionaryPath); + FileHelper.deleteCacheFile(HanLP.Config.CustomDictionaryPath); FileHelper.resetCustomPath(getDynamicCustomDictionary()); } // 3.clear trie @@ -110,9 +114,11 @@ public class HanlpHelper { } String hanlpPropertiesPath = getHanlpPropertiesPath(); - CustomDictionaryPath = Arrays.stream(CustomDictionaryPath).map(path -> hanlpPropertiesPath + FILE_SPILT + path) + HanLP.Config.CustomDictionaryPath = Arrays.stream(HanLP.Config.CustomDictionaryPath) + .map(path -> hanlpPropertiesPath + FILE_SPILT + path) .toArray(String[]::new); - log.info("hanlpPropertiesPath:{},CustomDictionaryPath:{}", hanlpPropertiesPath, CustomDictionaryPath); + log.info("hanlpPropertiesPath:{},CustomDictionaryPath:{}", hanlpPropertiesPath, + HanLP.Config.CustomDictionaryPath); HanLP.Config.CoreDictionaryPath = hanlpPropertiesPath + FILE_SPILT + HanLP.Config.BiGramDictionaryPath; HanLP.Config.CoreDictionaryTransformMatrixDictionaryPath = hanlpPropertiesPath + FILE_SPILT @@ -206,10 +212,18 @@ public class HanlpHelper { } } - public static List getTerms(String text) { + public static List getTerms(String text) { return getSegment().seg(text.toLowerCase()).stream() .filter(term -> term.getNature().startsWith(DictWordType.NATURE_SPILT)) + .map(term -> transform2ApiTerm(term)) .collect(Collectors.toList()); } + public static S2Term transform2ApiTerm(Term term) { + S2Term knowledgeTerm = new S2Term(); + BeanUtils.copyProperties(term, knowledgeTerm); + knowledgeTerm.setFrequency(term.getFrequency()); + return knowledgeTerm; + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/HdfsFileHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HdfsFileHelper.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/HdfsFileHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HdfsFileHelper.java index d429e8c57..b9eddd026 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/HdfsFileHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HdfsFileHelper.java @@ -1,7 +1,8 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.knowledge.helper; import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; +import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.utility.Predefine; import java.io.IOException; @@ -56,7 +57,7 @@ public class HdfsFileHelper { * @throws IOException */ public static void resetCustomPath(DynamicCustomDictionary customDictionary) throws IOException { - String[] path = CustomDictionaryPath; + String[] path = HanLP.Config.CustomDictionaryPath; FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration()); String cacheFilePath = path[0] + Predefine.BIN_EXT; int customBase = cacheFilePath.lastIndexOf(FileHelper.FILE_SPILT); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java index abd72eaf7..a37d35b92 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.knowledge.helper; import com.hankcs.hanlp.corpus.tag.Nature; -import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat; import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -53,7 +53,7 @@ public class NatureHelper { return result; } - private static boolean isViewOrEntity(Term term, Integer model) { + private static boolean isViewOrEntity(S2Term term, Integer model) { return (DictWordType.NATURE_SPILT + model).equals(term.nature.toString()) || term.nature.toString() .endsWith(DictWordType.ENTITY.getType()); } @@ -96,7 +96,7 @@ public class NatureHelper { && StringUtils.isNumeric(split[1]); } - public static ViewInfoStat getViewStat(List terms) { + public static ViewInfoStat getViewStat(List terms) { return ViewInfoStat.builder() .viewCount(getViewCount(terms)) .dimensionViewCount(getDimensionCount(terms)) @@ -105,20 +105,20 @@ public class NatureHelper { .build(); } - private static long getViewCount(List terms) { + private static long getViewCount(List terms) { return terms.stream().filter(term -> isViewOrEntity(term, getViewByNature(term.nature))).count(); } - private static long getDimensionValueCount(List terms) { + private static long getDimensionValueCount(List terms) { return terms.stream().filter(term -> isDimensionValueViewId(term.nature.toString())).count(); } - private static long getDimensionCount(List terms) { + private static long getDimensionCount(List terms) { return terms.stream().filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT) && term.nature.toString() .endsWith(DictWordType.DIMENSION.getType())).count(); } - private static long getMetricCount(List terms) { + private static long getMetricCount(List terms) { return terms.stream().filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT) && term.nature.toString() .endsWith(DictWordType.METRIC.getType())).count(); } @@ -130,7 +130,7 @@ public class NatureHelper { * @param terms * @return */ - public static Map> getViewToNatureStat(List terms) { + public static Map> getViewToNatureStat(List terms) { Map> modelToNature = new HashMap<>(); terms.stream().filter( term -> term.nature.startsWith(DictWordType.NATURE_SPILT) @@ -157,7 +157,7 @@ public class NatureHelper { return modelToNature; } - public static List selectPossibleViews(List terms) { + public static List selectPossibleViews(List terms) { Map> modelToNatureStat = getViewToNatureStat(terms); Integer maxViewTypeSize = modelToNatureStat.entrySet().stream() .max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size()) diff --git a/headless/core/src/main/resources/META-INF/spring.factories b/headless/core/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..bde7179af --- /dev/null +++ b/headless/core/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +com.tencent.supersonic.headless.core.file.FileHandler=\ + com.tencent.supersonic.headless.core.file.FileHandlerImpl \ No newline at end of file diff --git a/headless/server/pom.xml b/headless/server/pom.xml index a009887bf..b5950ff54 100644 --- a/headless/server/pom.xml +++ b/headless/server/pom.xml @@ -17,9 +17,6 @@ UTF-8 - - - org.projectlombok @@ -115,6 +112,8 @@ postgresql ${postgresql.version} + + \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java new file mode 100644 index 000000000..61316dc79 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java @@ -0,0 +1,22 @@ +package com.tencent.supersonic.headless.server.persistence.dataobject; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.util.Date; + +@Data +@TableName("s2_dictionary_conf") +public class DictConfDO { + @TableId(type = IdType.AUTO) + private Long id; + private String description; + private String type; + private Long itemId; + private String config; + private String status; + private Date createdAt; + private String createdBy; +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java new file mode 100644 index 000000000..25ee9db92 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java @@ -0,0 +1,25 @@ +package com.tencent.supersonic.headless.server.persistence.dataobject; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.util.Date; + +@Data +@TableName("s2_dictionary_task") +public class DictTaskDO { + @TableId(type = IdType.AUTO) + private Long id; + private String name; + private String description; + private String type; + private Long itemId; + private String config; + private String status; + private Date createdAt; + private String createdBy; + private Long elapsedMs; + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java new file mode 100644 index 000000000..b3e9f890b --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java @@ -0,0 +1,9 @@ +package com.tencent.supersonic.headless.server.persistence.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface DictConfMapper extends BaseMapper { +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java new file mode 100644 index 000000000..a13c04055 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.headless.server.persistence.mapper; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface DictTaskMapper extends BaseMapper { + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DictRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DictRepository.java new file mode 100644 index 000000000..37008166c --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DictRepository.java @@ -0,0 +1,29 @@ +package com.tencent.supersonic.headless.server.persistence.repository; + +import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; +import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; + +import java.util.List; + +public interface DictRepository { + + Long addDictConf(DictConfDO dictConfDO); + + Long editDictConf(DictConfDO dictConfDO); + + List queryDictConf(DictItemFilter dictItemFilter); + + Long addDictTask(DictTaskDO dictTaskDO); + + Long editDictTask(DictTaskDO dictTaskDO); + + DictTaskDO queryDictTask(DictItemFilter filter); + + DictTaskDO queryDictTaskById(Long id); + + DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java new file mode 100644 index 000000000..020992ef8 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java @@ -0,0 +1,142 @@ +package com.tencent.supersonic.headless.server.persistence.repository.impl; + + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; +import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; +import com.tencent.supersonic.headless.server.persistence.mapper.DictConfMapper; +import com.tencent.supersonic.headless.server.persistence.mapper.DictTaskMapper; +import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; +import com.tencent.supersonic.headless.server.service.DimensionService; +import com.tencent.supersonic.headless.server.utils.DictUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Repository; +import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.Objects; + +@Slf4j +@Repository +public class DictRepositoryImpl implements DictRepository { + + private final DictTaskMapper dictTaskMapper; + private final DictConfMapper dictConfMapper; + private final DictUtils dictConverter; + private final DimensionService dimensionService; + + public DictRepositoryImpl(DictTaskMapper dictTaskMapper, DictConfMapper dictConfMapper, + DictUtils dictConverter, DimensionService dimensionService) { + this.dictTaskMapper = dictTaskMapper; + this.dictConfMapper = dictConfMapper; + this.dictConverter = dictConverter; + this.dimensionService = dimensionService; + } + + @Override + public Long addDictTask(DictTaskDO dictTaskDO) { + dictTaskMapper.insert(dictTaskDO); + return dictTaskDO.getId(); + } + + @Override + public Long editDictTask(DictTaskDO dictTaskDO) { + dictTaskMapper.updateById(dictTaskDO); + return dictTaskDO.getId(); + } + + @Override + public DictTaskDO queryDictTask(DictItemFilter filter) { + QueryWrapper wrapper = new QueryWrapper<>(); + if (Objects.nonNull(filter.getItemId())) { + wrapper.lambda().eq(DictTaskDO::getItemId, filter.getItemId()); + } + if (Objects.nonNull(filter.getType())) { + wrapper.lambda().eq(DictTaskDO::getType, filter.getType()); + } + if (Objects.nonNull(filter.getId())) { + wrapper.lambda().eq(DictTaskDO::getId, filter.getId()); + } + + List dictTaskDOList = dictTaskMapper.selectList(wrapper); + if (CollectionUtils.isEmpty(dictTaskDOList)) { + return null; + } + return dictTaskDOList.get(0); + } + + @Override + public DictTaskDO queryDictTaskById(Long id) { + return dictTaskMapper.selectById(id); + } + + @Override + public DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq) { + DictTaskResp taskResp = new DictTaskResp(); + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId()); + wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType()); + List dictTaskDOList = dictTaskMapper.selectList(wrapper); + if (CollectionUtils.isEmpty(dictTaskDOList)) { + return taskResp; + } + taskResp = dictConverter.taskDO2Resp(dictTaskDOList.get(0)); + DimensionResp dimension = dimensionService.getDimension(taskReq.getItemId()); + taskResp.setBizName(dimension.getBizName()); + taskResp.setModelId(dimension.getModelId()); + return taskResp; + } + + @Override + public Long addDictConf(DictConfDO dictConfDO) { + dictConfMapper.insert(dictConfDO); + return dictConfDO.getId(); + } + + @Override + public Long editDictConf(DictConfDO dictConfDO) { + DictItemFilter filter = DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType())) + .itemId(dictConfDO.getItemId()) + .status(StatusEnum.ONLINE) + .build(); + + List dictConfDOList = getDictConfDOList(filter); + if (CollectionUtils.isEmpty(dictConfDOList)) { + return -1L; + } + dictConfDO.setId(dictConfDOList.get(0).getId()); + dictConfMapper.updateById(dictConfDO); + return dictConfDO.getId(); + } + + @Override + public List queryDictConf(DictItemFilter dictItemFilter) { + List dictConfDOList = getDictConfDOList(dictItemFilter); + return dictConverter.dictDOList2Req(dictConfDOList); + } + + private List getDictConfDOList(DictItemFilter dictItemFilter) { + QueryWrapper wrapper = new QueryWrapper<>(); + if (Objects.nonNull(dictItemFilter.getId())) { + wrapper.lambda().eq(DictConfDO::getId, dictItemFilter.getId()); + } + if (Objects.nonNull(dictItemFilter.getType())) { + wrapper.lambda().eq(DictConfDO::getType, dictItemFilter.getType().name()); + } + if (Objects.nonNull(dictItemFilter.getItemId())) { + wrapper.lambda().eq(DictConfDO::getItemId, dictItemFilter.getItemId()); + } + if (Objects.nonNull(dictItemFilter.getStatus())) { + wrapper.lambda().eq(DictConfDO::getStatus, dictItemFilter.getStatus().name()); + } + List dictConfDOList = dictConfMapper.selectList(wrapper); + return dictConfDOList; + } +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java new file mode 100644 index 000000000..b6e2d65e6 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java @@ -0,0 +1,132 @@ +package com.tencent.supersonic.headless.server.rest; + + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; + +import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; +import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; +import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.server.service.DictConfService; +import com.tencent.supersonic.headless.server.service.DictTaskService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + + +@RestController +@RequestMapping("/api/semantic/knowledge") +public class KnowledgeController { + + @Autowired + private DictTaskService taskService; + + @Autowired + private DictConfService confService; + + /** + * addDictConf-新增item的字典配置 + * Add configuration information for dictionary entries + * + * @param dictItemReq + */ + @PostMapping("/conf") + public Long addDictConf(@RequestBody @Valid DictItemReq dictItemReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return confService.addDictConf(dictItemReq, user); + } + + /** + * editDictConf-编辑item的字典配置 + * Edit configuration information for dictionary entries + * + * @param dictItemReq + */ + @PutMapping("/conf") + public Long editDictConf(@RequestBody @Valid DictItemReq dictItemReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return confService.editDictConf(dictItemReq, user); + } + + /** + * queryDictConf-查询item的字典配置 + * query configuration information for dictionary entries + * + * @param filter + */ + @PostMapping("/conf/query") + public List queryDictConf(@RequestBody @Valid DictItemFilter filter, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return confService.queryDictConf(filter, user); + } + + + /** + * addDictTask-实时导入一个item的字典数据 + * write specific item values to the knowledge base + * + * @param taskReq + */ + @PostMapping("/task") + public Long addDictTask(@RequestBody DictSingleTaskReq taskReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return taskService.addDictTask(taskReq, user); + } + + /** + * deleteDictTask-实时删除某一个item的字典数据 + * remove specific item values from the knowledge base + * + * @param taskReq + */ + @PutMapping("/task/delete") + public Long deleteDictTask(@RequestBody DictSingleTaskReq taskReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return taskService.deleteDictTask(taskReq, user); + } + + /** + * dailyDictTask-手动离线更新所以字典 + */ + @PutMapping("/task/all") + public Boolean dailyDictTask( + HttpServletRequest request, + HttpServletResponse response) { + return taskService.dailyDictTask(); + } + + /** + * queryLatestDictTask-返回最新的字典任务执行情况 + * + * @param taskReq + */ + @PostMapping("/task/search") + public DictTaskResp queryLatestDictTask(@RequestBody DictSingleTaskReq taskReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return taskService.queryLatestDictTask(taskReq, user); + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java new file mode 100644 index 000000000..1dff03d54 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; +import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; + +import java.util.List; + +/** + * Make relevant settings for the dictionary + */ +public interface DictConfService { + + Long addDictConf(DictItemReq itemValueReq, User user); + + Long editDictConf(DictItemReq itemValueReq, User user); + + List queryDictConf(DictItemFilter dictItemFilter, User user); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java new file mode 100644 index 000000000..e7e7c65e5 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; + +/** + * Manage dictionary tasks + */ +public interface DictTaskService { + Long addDictTask(DictSingleTaskReq taskReq, User user); + + Long deleteDictTask(DictSingleTaskReq taskReq, User user); + + Boolean dailyDictTask(); + + DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java new file mode 100644 index 000000000..2b3598b33 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java @@ -0,0 +1,25 @@ +package com.tencent.supersonic.headless.server.service; + + +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; + +import java.util.List; +import java.util.Set; + +public interface KnowledgeService { + + List getTerms(String text); + + List prefixSearch(String key, int limit, Set viewIds); + + List suffixSearch(String key, int limit, Set viewIds); + + void updateSemanticKnowledge(List natures); + + void reloadAllData(List natures); + + void updateOnlineKnowledge(List natures); + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java new file mode 100644 index 000000000..14750879a --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java @@ -0,0 +1,50 @@ +package com.tencent.supersonic.headless.server.service.impl; + + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; +import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; +import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; +import com.tencent.supersonic.headless.server.service.DictConfService; +import com.tencent.supersonic.headless.server.utils.DictUtils; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class DictConfServiceImpl implements DictConfService { + + private final DictRepository dictRepository; + private final DictUtils dictConverter; + + public DictConfServiceImpl(DictRepository dictRepository, + DictUtils dictConverter) { + this.dictRepository = dictRepository; + this.dictConverter = dictConverter; + } + + @Override + public Long addDictConf(DictItemReq itemValueReq, User user) { + DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user); + return dictRepository.addDictConf(dictConfDO); + } + + @Override + public Long editDictConf(DictItemReq itemValueReq, User user) { + DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user); + dictRepository.editDictConf(dictConfDO); + if (StatusEnum.DELETED.equals(itemValueReq.getStatus())) { + // todo delete dict file and refresh + + } + return itemValueReq.getItemId(); + } + + @Override + public List queryDictConf(DictItemFilter dictItemFilter, User user) { + return dictRepository.queryDictConf(dictItemFilter); + } +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java new file mode 100644 index 000000000..79a8755d9 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -0,0 +1,150 @@ +package com.tencent.supersonic.headless.server.service.impl; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; +import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.core.file.FileHandler; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; +import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; +import com.tencent.supersonic.headless.server.service.DictTaskService; +import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.server.utils.DictUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +@Service +@Slf4j +public class DictTaskServiceImpl implements DictTaskService { + + @Value("${dict.flush.enable:true}") + private Boolean dictFlushEnable; + @Value("${dict.flush.daily.enable:true}") + private Boolean dictFlushDailyEnable; + @Value("${dict.file.type:txt}") + private String dictFileType; + private String dimValue = "DimValue_%d_%d"; + + private final DictRepository dictRepository; + private final DictUtils dictConverter; + private final DictUtils dictUtils; + private final FileHandler fileHandler; + private final KnowledgeService knowledgeService; + + public DictTaskServiceImpl(DictRepository dictRepository, + DictUtils dictConverter, + DictUtils dictUtils, + FileHandler fileHandler, + KnowledgeService knowledgeService) { + this.dictRepository = dictRepository; + this.dictConverter = dictConverter; + this.dictUtils = dictUtils; + this.fileHandler = fileHandler; + this.knowledgeService = knowledgeService; + } + + @Override + public Long addDictTask(DictSingleTaskReq taskReq, User user) { + if (!dictFlushEnable) { + return 0L; + } + DictItemResp dictItemResp = fetchDictItemResp(taskReq); + return handleDictTaskByItemResp(dictItemResp, user); + } + + private Long handleDictTaskByItemResp(DictItemResp dictItemResp, User user) { + DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user); + log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); + dictRepository.addDictTask(dictTaskDO); + Long idInDb = dictTaskDO.getId(); + dictItemResp.setId(idInDb); + runDictTask(dictItemResp, user); + return idInDb; + } + + private DictItemResp fetchDictItemResp(DictSingleTaskReq taskReq) { + DictItemFilter dictItemFilter = DictItemFilter.builder() + .itemId(taskReq.getItemId()) + .type(taskReq.getType()) + .build(); + List dictItemRespList = dictRepository.queryDictConf(dictItemFilter); + if (!CollectionUtils.isEmpty(dictItemRespList)) { + return dictItemRespList.get(0); + } + return null; + } + + private void runDictTask(DictItemResp dictItemResp, User user) { + if (Objects.isNull(dictItemResp)) { + return; + } + + DictTaskDO dictTaskDO = dictRepository.queryDictTaskById(dictItemResp.getId()); + dictTaskDO.setStatus(TaskStatusEnum.RUNNING.getStatus()); + dictRepository.editDictTask(dictTaskDO); + + // 1.生成item字典数据 + List data = dictUtils.fetchItemValue(dictItemResp); + + // 2.变更字典文件 + String fileName = dictItemResp.fetchDictFileName() + Constants.DOT + dictFileType; + fileHandler.writeFile(data, fileName, false); + + // 3.实时变更内存中字典数据 + try { + HanlpHelper.reloadCustomDictionary(); + dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); + dictRepository.editDictTask(dictTaskDO); + } catch (IOException e) { + log.error("reloadCustomDictionary error", e); + } + + } + + @Override + public Long deleteDictTask(DictSingleTaskReq taskReq, User user) { + DictItemResp dictItemResp = fetchDictItemResp(taskReq); + String fileName = dictItemResp.fetchDictFileName() + Constants.DOT + dictFileType; + fileHandler.deleteDictFile(fileName); + + try { + HanlpHelper.reloadCustomDictionary(); + } catch (Exception e) { + log.error("reloadCustomDictionary error", e); + } + + return 0L; + } + + @Override + @Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}") + public Boolean dailyDictTask() { + log.info("[dailyDictTask] start"); + if (!dictFlushDailyEnable) { + log.info("dictFlushDailyEnable is false, now finish dailyDictTask"); + } + DictItemFilter filter = DictItemFilter.builder().status(StatusEnum.ONLINE).build(); + List dictItemRespList = dictRepository.queryDictConf(filter); + dictItemRespList.stream().forEach(item -> handleDictTaskByItemResp(item, null)); + log.info("[dailyDictTask] finish"); + return true; + } + + @Override + public DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user) { + return dictRepository.queryLatestDictTask(taskReq); + } + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java new file mode 100644 index 000000000..58ff737b1 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java @@ -0,0 +1,107 @@ +package com.tencent.supersonic.headless.server.service.impl; + +import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.SearchService; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.server.service.ViewService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class KnowledgeServiceImpl implements KnowledgeService { + + private final ViewService viewService; + + public KnowledgeServiceImpl(ViewService viewService) { + this.viewService = viewService; + } + + @Override + public void updateSemanticKnowledge(List natures) { + + List prefixes = natures.stream() + .filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType())) + .collect(Collectors.toList()); + + for (DictWord nature : prefixes) { + HanlpHelper.addToCustomDictionary(nature); + } + + List suffixes = natures.stream() + .filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType())) + .collect(Collectors.toList()); + + SearchService.loadSuffix(suffixes); + } + + @Override + public void reloadAllData(List natures) { + // 1. reload custom knowledge + try { + HanlpHelper.reloadCustomDictionary(); + } catch (Exception e) { + log.error("reloadCustomDictionary error", e); + } + + // 2. update online knowledge + updateOnlineKnowledge(natures); + } + + @Override + public void updateOnlineKnowledge(List natures) { + try { + updateSemanticKnowledge(natures); + } catch (Exception e) { + log.error("updateSemanticKnowledge error", e); + } + } + + @Override + public List getTerms(String text) { + return HanlpHelper.getTerms(text); + } + + @Override + public List prefixSearch(String key, int limit, Set viewIds) { + return prefixSearch(key, limit, viewIds2ModelIdList(viewIds)); + } + + public List prefixSearchByModel(String key, int limit, Set models) { + return SearchService.prefixSearch(key, limit, models); + } + + @Override + public List suffixSearch(String key, int limit, Set viewIds) { + return suffixSearch(key, limit, viewIds2ModelIdList(viewIds)); + } + + public List suffixSearchByModel(String key, int limit, Set models) { + return SearchService.suffixSearch(key, limit, models); + } + + private Set viewIds2ModelIdList(Set viewIds) { + Set modelIds = new HashSet<>(); + MetaFilter filter = new MetaFilter(); + filter.setIds(new ArrayList<>(viewIds)); + List viewList = viewService.getViewList(filter); + if (CollectionUtils.isEmpty(viewList)) { + return modelIds; + } + viewList.stream().forEach(view -> modelIds.addAll(view.getAllModels())); + return modelIds; + } +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java new file mode 100644 index 000000000..4de9272a4 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -0,0 +1,356 @@ +package com.tencent.supersonic.headless.server.utils; + +import com.google.common.base.Strings; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.Filter; +import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; +import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; +import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; +import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; +import com.tencent.supersonic.headless.server.service.DimensionService; +import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.QueryService; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; + +import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; +import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.SPACE; + +@Component +public class DictUtils { + + private static String dateTimeFormatter = "yyyyMMddHHmmss"; + @Value("${dimension.multi.value.split:#}") + private String dimMultiValueSplit; + + @Value("${item.value.max.count:100000}") + private Long itemValueMaxCount; + + @Value("${item.value.white.frequency:999999}") + private Long itemValueWhiteFrequency; + + @Value("${item.value.date.start:1}") + private Integer itemValueDateStart; + @Value("${item.value.date.end:1}") + private Integer itemValueDateEnd; + + + private final DimensionService dimensionService; + private final MetricService metricService; + private final QueryService queryService; + private final ModelService modelService; + + public DictUtils(DimensionService dimensionService, + MetricService metricService, + QueryService queryService, + ModelService modelService) { + this.dimensionService = dimensionService; + this.metricService = metricService; + this.queryService = queryService; + this.modelService = modelService; + } + + public String fetchDictFileName(DictItemResp dictItemResp) { + return String.format("dic_value_%d_%s_%s", dictItemResp.getModelId(), dictItemResp.getType().name(), + dictItemResp.getItemId()); + } + + public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user) { + DictTaskDO taskDO = new DictTaskDO(); + Date createAt = new Date(); + String name = dictItemResp.fetchDictFileName(); + taskDO.setName(name); + taskDO.setType(dictItemResp.getType().name()); + taskDO.setItemId(dictItemResp.getItemId()); + taskDO.setConfig(JsonUtil.toString(dictItemResp.getConfig())); + taskDO.setStatus(TaskStatusEnum.PENDING.getStatus()); + taskDO.setCreatedAt(createAt); + String creator = (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) ? "" : user.getName(); + taskDO.setCreatedBy(creator); + return taskDO; + } + + public DictConfDO generateDictConfDO(DictItemReq itemValueReq, User user) { + DictConfDO confDO = new DictConfDO(); + BeanUtils.copyProperties(itemValueReq, confDO); + confDO.setType(itemValueReq.getType().name()); + confDO.setConfig(JsonUtil.toString(itemValueReq.getConfig())); + Date createAt = new Date(); + confDO.setCreatedAt(createAt); + String creator = Strings.isNullOrEmpty(user.getName()) ? "" : user.getName(); + confDO.setCreatedBy(creator); + confDO.setStatus(itemValueReq.getStatus().name()); + return confDO; + } + + public List dictDOList2Req(List dictConfDOList) { + List dictItemReqList = new ArrayList<>(); + dictConfDOList.stream().forEach(conf -> dictItemReqList.add(dictDO2Req(conf))); + return dictItemReqList; + } + + public DictItemResp dictDO2Req(DictConfDO dictConfDO) { + DictItemResp dictItemResp = new DictItemResp(); + BeanUtils.copyProperties(dictConfDO, dictItemResp); + dictItemResp.setType(TypeEnums.valueOf(dictConfDO.getType())); + dictItemResp.setConfig(JsonUtil.toObject(dictConfDO.getConfig(), ItemValueConfig.class)); + dictItemResp.setStatus(StatusEnum.of(dictConfDO.getStatus())); + if (TypeEnums.DIMENSION.equals(TypeEnums.valueOf(dictConfDO.getType()))) { + DimensionResp dimension = dimensionService.getDimension(dictConfDO.getItemId()); + dictItemResp.setModelId(dimension.getModelId()); + dictItemResp.setBizName(dimension.getBizName()); + } + + return dictItemResp; + } + + public List fetchItemValue(DictItemResp dictItemResp) { + List lines = new ArrayList<>(); + QuerySqlReq querySqlReq = constructQueryReq(dictItemResp); + querySqlReq.setNeedAuth(false); + String bizName = dictItemResp.getBizName(); + try { + SemanticQueryResp semanticQueryResp = queryService.queryByReq(querySqlReq, null); + if (Objects.isNull(semanticQueryResp) || CollectionUtils.isEmpty(semanticQueryResp.getResultList())) { + return lines; + } + Map valueAndFrequencyPair = new HashMap<>(2000); + for (Map line : semanticQueryResp.getResultList()) { + + if (CollectionUtils.isEmpty(line) || !line.containsKey(bizName) + || line.get(bizName) == null || line.size() != 2) { + continue; + } + String dimValue = line.get(bizName).toString(); + Object metricObject = null; + for (String key : line.keySet()) { + if (!bizName.equalsIgnoreCase(key)) { + metricObject = line.get(key); + } + } + if (!Strings.isNullOrEmpty(dimValue) && Objects.nonNull(metricObject)) { + Long metric = Math.round(Double.parseDouble(metricObject.toString())); + mergeMultivaluedValue(valueAndFrequencyPair, dimValue, metric); + } + } + String nature = dictItemResp.getNature(); + constructDictLines(valueAndFrequencyPair, lines, nature); + addWhiteValueLines(dictItemResp, lines, nature); + } catch (Exception e) { + e.printStackTrace(); + } + return lines; + } + + private void addWhiteValueLines(DictItemResp dictItemResp, List lines, String nature) { + if (Objects.isNull(dictItemResp) || Objects.isNull(dictItemResp.getConfig()) + || CollectionUtils.isEmpty(dictItemResp.getConfig().getWhiteList())) { + return; + } + List whiteList = dictItemResp.getConfig().getWhiteList(); + whiteList.forEach(white -> lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency))); + } + + private void constructDictLines(Map valueAndFrequencyPair, List lines, String nature) { + if (CollectionUtils.isEmpty(valueAndFrequencyPair)) { + return; + } + + valueAndFrequencyPair.forEach((value, frequency) -> { + lines.add(String.format("%s %s %s", value, nature, frequency)); + }); + } + + private void mergeMultivaluedValue(Map valueAndFrequencyPair, String dimValue, Long metric) { + if (org.apache.logging.log4j.util.Strings.isEmpty(dimValue)) { + return; + } + Map tmp = new HashMap<>(); + if (dimValue.contains(dimMultiValueSplit)) { + Arrays.stream(dimValue.split(dimMultiValueSplit)) + .forEach(dimValueSingle -> tmp.put(dimValueSingle, metric)); + } else { + tmp.put(dimValue, metric); + } + + for (String value : tmp.keySet()) { + long metricOld = valueAndFrequencyPair.containsKey(value) ? valueAndFrequencyPair.get(value) : 0L; + valueAndFrequencyPair.put(value, metric + metricOld); + } + } + + private QuerySqlReq constructQueryReq(DictItemResp dictItemResp) { + if (Objects.nonNull(dictItemResp) && Objects.nonNull(dictItemResp.getConfig()) + && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { + // 查询默认指标 + QueryStructReq queryStructReq = generateQueryStruct(dictItemResp); + return queryStructReq.convert(queryStructReq, true); + } + // count(1) 作为指标 + return constructQuerySqlReq(dictItemResp); + } + + private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { + String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d"; + String bizName = dictItemResp.getBizName(); + String whereStr = generateWhereStr(dictItemResp); + String where = Strings.isNullOrEmpty(whereStr) ? "" : "WHERE" + whereStr; + ItemValueConfig config = dictItemResp.getConfig(); + Long limit = (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount : + dictItemResp.getConfig().getLimit(); + String sql = String.format(sqlPattern, bizName, where, bizName, limit); + Set modelIds = new HashSet<>(); + modelIds.add(dictItemResp.getModelId()); + QuerySqlReq querySqlReq = new QuerySqlReq(); + querySqlReq.setSql(sql); + querySqlReq.setNeedAuth(false); + querySqlReq.setModelIds(modelIds); + + return querySqlReq; + } + + private QueryStructReq generateQueryStruct(DictItemResp dictItemResp) { + QueryStructReq queryStructReq = new QueryStructReq(); + + Set modelIds = new HashSet<>(Arrays.asList(dictItemResp.getModelId())); + queryStructReq.setModelIds(modelIds); + + List groups = new ArrayList<>(Arrays.asList(dictItemResp.getBizName())); + queryStructReq.setGroups(groups); + + List filters = generateFilters(dictItemResp); + queryStructReq.setDimensionFilters(filters); + + List aggregators = new ArrayList<>(); + Long metricId = dictItemResp.getConfig().getMetricId(); + MetricResp metric = metricService.getMetric(metricId); + String metricBizName = metric.getBizName(); + aggregators.add(new Aggregator(metricBizName, AggOperatorEnum.SUM)); + queryStructReq.setAggregators(aggregators); + + List orders = new ArrayList<>(); + orders.add(new Order(metricBizName, Constants.DESC_UPPER)); + queryStructReq.setOrders(orders); + + fillStructDateInfo(queryStructReq, dictItemResp); + + Long limit = Objects.isNull(dictItemResp.getConfig().getLimit()) ? itemValueMaxCount : + dictItemResp.getConfig().getLimit(); + queryStructReq.setLimit(limit); + queryStructReq.setNeedAuth(false); + return queryStructReq; + } + + private void fillStructDateInfo(QueryStructReq queryStructReq, DictItemResp dictItemResp) { + ModelResp model = modelService.getModel(dictItemResp.getModelId()); + if (Objects.nonNull(model)) { + List timeDims = model.getTimeDimension(); + if (!CollectionUtils.isEmpty(timeDims)) { + DateConf dateConf = new DateConf(); + dateConf.setDateMode(DateConf.DateMode.BETWEEN); + String format = timeDims.get(0).getDateFormat(); + String start = LocalDate.now().minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(format)); + String end = LocalDate.now().minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(format)); + dateConf.setStartDate(start); + dateConf.setEndDate(end); + queryStructReq.setDateInfo(dateConf); + } + } + } + + private List generateFilters(DictItemResp dictItemResp) { + List filters = new ArrayList<>(); + if (Objects.isNull(dictItemResp)) { + return new ArrayList<>(); + } + String whereStr = generateWhereStr(dictItemResp); + if (Strings.isNullOrEmpty(whereStr)) { + return new ArrayList<>(); + } + Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, whereStr); + filters.add(filter); + return filters; + } + + public String generateWhereStr(DictItemResp dictItemResp) { + StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE); + + String bizName = dictItemResp.getBizName(); + ItemValueConfig config = dictItemResp.getConfig(); + if (Objects.nonNull(config)) { + if (!CollectionUtils.isEmpty(config.getBlackList())) { + StringJoiner joinerBlack = new StringJoiner(COMMA); + config.getBlackList().stream().forEach(black -> joinerBlack.add(APOSTROPHE + black + APOSTROPHE)); + joiner.add(String.format("(%s not in (%s))", bizName, joinerBlack.toString())); + } + + if (!CollectionUtils.isEmpty(config.getRuleList())) { + config.getRuleList().stream().forEach(rule -> joiner.add("(" + rule + ")")); + } + } + + ModelResp model = modelService.getModel(dictItemResp.getModelId()); + if (Objects.nonNull(model)) { + List timeDims = model.getTimeDimension(); + if (!CollectionUtils.isEmpty(timeDims)) { + String format = timeDims.get(0).getDateFormat(); + String start = LocalDate.now().minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(format)); + String end = LocalDate.now().minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(format)); + joiner.add(String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), start, + TimeDimensionEnum.DAY.getName(), end)); + } + } + return joiner.toString(); + } + + public DictTaskResp taskDO2Resp(DictTaskDO dictTaskDO) { + DictTaskResp resp = new DictTaskResp(); + BeanUtils.copyProperties(dictTaskDO, resp); + resp.setTaskStatus(dictTaskDO.getStatus()); + resp.setType(TypeEnums.valueOf(dictTaskDO.getType())); + resp.setConfig(JsonUtil.toObject(dictTaskDO.getConfig(), ItemValueConfig.class)); + return resp; + } +} \ No newline at end of file diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index 7f26866e9..bb375e249 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -19,8 +19,8 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \ com.tencent.supersonic.chat.core.corrector.HavingCorrector -com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\ - com.tencent.supersonic.chat.core.knowledge.semantic.RemoteSemanticInterpreter +com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\ + com.tencent.supersonic.chat.core.query.semantic.RemoteSemanticInterpreter com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver diff --git a/launchers/chat/src/main/resources/db/chat-schema-h2.sql b/launchers/chat/src/main/resources/db/chat-schema-h2.sql index 20e5c3bab..a296a0a8f 100644 --- a/launchers/chat/src/main/resources/db/chat-schema-h2.sql +++ b/launchers/chat/src/main/resources/db/chat-schema-h2.sql @@ -156,23 +156,23 @@ CREATE TABLE IF NOT EXISTS `s2_user_department` ( ); COMMENT ON TABLE s2_user_department IS 'user_department_info'; - -CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( - `id` INT NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL , -- task name - `description` varchar(255) , - `command`LONGVARCHAR NOT NULL , -- task Request Parameters - `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 - `status` INT NOT NULL , -- the final status of the task - `dimension_ids` varchar(500) NULL , - `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , - `created_by` varchar(100) NOT NULL , - `progress` DOUBLE default 0.00 , -- task real-time progress - `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds - `message` LONGVARCHAR , -- remark related information - PRIMARY KEY (`id`) - ); -COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; +-- +-- CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( +-- `id` INT NOT NULL AUTO_INCREMENT, +-- `name` varchar(255) NOT NULL , -- task name +-- `description` varchar(255) , +-- `command`LONGVARCHAR NOT NULL , -- task Request Parameters +-- `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 +-- `status` INT NOT NULL , -- the final status of the task +-- `dimension_ids` varchar(500) NULL , +-- `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , +-- `created_by` varchar(100) NOT NULL , +-- `progress` DOUBLE default 0.00 , -- task real-time progress +-- `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds +-- `message` LONGVARCHAR , -- remark related information +-- PRIMARY KEY (`id`) +-- ); +-- COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index 84cc2326e..240dc7e28 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -23,6 +23,7 @@ import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.server.service.KnowledgeService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -30,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import java.util.Arrays; import java.util.List; @@ -51,6 +53,8 @@ public class ChatDemoLoader implements CommandLineRunner { private AgentService agentService; @Autowired private SysParameterService sysParameterService; + @Autowired + private KnowledgeService knowledgeService; @Value("${demo.enabled:false}") private boolean demoEnabled; @@ -91,7 +95,10 @@ public class ChatDemoLoader implements CommandLineRunner { queryRequest.setAgentId(1); queryRequest.setUser(User.getFakeUser()); ParseResp parseResp = queryService.performParsing(queryRequest); - + if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) { + log.info("parseResp.getSelectedParses() is empty"); + return; + } ExecuteQueryReq executeReq = ExecuteQueryReq.builder().build(); executeReq.setQueryId(parseResp.getQueryId()); executeReq.setParseId(parseResp.getSelectedParses().get(0).getId()); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 07f97aed2..a8fbd7053 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -28,8 +28,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor, \ com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor -com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\ - com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter +com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\ + com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 5b7ac3b30..b9558f453 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -393,19 +393,30 @@ CREATE TABLE IF NOT EXISTS `singer` ( ); COMMENT ON TABLE singer IS 'singer_info'; +CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` ( + `id` INT NOT NULL AUTO_INCREMENT, + `description` varchar(255) , + `type` varchar(255) NOT NULL , + `item_id` INT NOT NULL , -- task Request Parameters md5 + `config` LONGVARCHAR , -- remark related information + `status` varchar(255) NOT NULL , -- the final status of the task + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `created_by` varchar(100) NOT NULL , + PRIMARY KEY (`id`) +); +COMMENT ON TABLE s2_dictionary_conf IS 'dictionary conf information table'; + CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( `id` INT NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL , -- task name `description` varchar(255) , - `command`LONGVARCHAR NOT NULL , -- task Request Parameters - `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 - `status` INT NOT NULL , -- the final status of the task - `dimension_ids` varchar(500) NULL , + `type` varchar(255) NOT NULL , + `item_id` INT NOT NULL , -- task Request Parameters md5 + `config` LONGVARCHAR , -- remark related information + `status` varchar(255) NOT NULL , -- the final status of the task `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `created_by` varchar(100) NOT NULL , - `progress` DOUBLE default 0.00 , -- task real-time progress `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds - `message` LONGVARCHAR , -- remark related information PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index ce2ef9633..76cc578f2 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -201,37 +201,33 @@ CREATE TABLE `s2_database` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='数据库实例表'; -CREATE TABLE `s2_dictionary` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, - `item_id` bigint(20) DEFAULT NULL COMMENT '对应维度id、指标id等', - `type` varchar(50) DEFAULT NULL COMMENT '对应维度、指标等', - `black_list` mediumtext COMMENT '字典黑名单', - `white_list` mediumtext COMMENT '字典白名单', - `rule_list` mediumtext COMMENT '字典规则', - `is_dict_Info` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-开启写入字典，0-不开启', - `created_at` datetime NOT NULL COMMENT '创建时间', - `updated_at` datetime NOT NULL COMMENT '更新时间', - `created_by` varchar(100) NOT NULL COMMENT '创建人', - `updated_by` varchar(100) DEFAULT NULL COMMENT '更新人', - `is_deleted` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-删除,0-可用', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='字典配置信息表'; +CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `description` varchar(255) , + `type` varchar(255) NOT NULL , + `item_id` INT NOT NULL , -- task Request Parameters md5 + `config` mediumtext , -- remark related information + `status` varchar(255) NOT NULL , -- the final status of the task + `created_at` datetime NOT NULL COMMENT '创建时间' , + `created_by` varchar(100) NOT NULL , + PRIMARY KEY (`id`) +); +COMMENT ON TABLE s2_dictionary_conf IS '字典配置信息表'; -CREATE TABLE `s2_dictionary_task` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务名称', - `description` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '任务描述', - `command` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数', - `command_md5` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数', - `dimension_ids` mediumtext NULL COMMENT '本次执行维度列表', - `status` int(10) NOT NULL COMMENT '任务最终运行状态', - `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `created_by` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '创建人', - `progress` double(3,2) DEFAULT '0.00' COMMENT '任务进度', - `elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时', - `message` mediumtext COLLATE utf8mb4_unicode_ci COMMENT '备注相关信息', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='字典任务信息表'; +CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL , -- task name + `description` varchar(255) , + `type` varchar(255) NOT NULL , + `item_id` INT NOT NULL , -- task Request Parameters md5 + `config` mediumtext , -- remark related information + `status` varchar(255) NOT NULL , -- the final status of the task + `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `created_by` varchar(100) NOT NULL , + `elapsed_ms` int(10) DEFAULT NULL , -- the task takes time in milliseconds + PRIMARY KEY (`id`) +); +COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; CREATE TABLE `s2_dimension` ( diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java index a80203908..b09408519 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat; import com.tencent.supersonic.BaseApplication; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java index 35e0afd68..1ee5fbd4c 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java index e87f7d513..5e21a0180 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.mapper; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java new file mode 100644 index 000000000..9eeaf0ba8 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java @@ -0,0 +1,94 @@ +package com.tencent.supersonic.headless; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; +import com.tencent.supersonic.headless.server.persistence.mapper.DictConfMapper; +import com.tencent.supersonic.headless.server.service.DictTaskService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + + +public class DictTest extends BaseTest { + @Autowired + private DictConfMapper confMapper; + + @Autowired + private DictTaskService taskService; + + @Test + public void insertConf() { + DictConfDO confDO = new DictConfDO(); + Date createAt = new Date(); + confDO.setType(TypeEnums.DIMENSION.name()); + confDO.setItemId(1L); + confDO.setConfig(JsonUtil.toString(new ItemValueConfig())); + confDO.setStatus(StatusEnum.ONLINE.getStatus()); + confDO.setCreatedAt(createAt); + confDO.setCreatedBy("admin"); + confMapper.insert(confDO); + DictConfDO confDODb = confMapper.selectById(1L); + System.out.println(confDODb.getId()); + } + + @Test + public void editConf() { + DictConfDO confDO = new DictConfDO(); + Date createAt = new Date(); + confDO.setType(TypeEnums.DIMENSION.name()); + confDO.setItemId(3L); + ItemValueConfig config = new ItemValueConfig(); + config.setMetricId(4L); + config.setBlackList(new ArrayList<>(Arrays.asList("1", "3"))); + config.setWhiteList(new ArrayList<>(Arrays.asList("2", "4"))); + confDO.setConfig(JsonUtil.toString(config)); + confDO.setStatus(TaskStatusEnum.PENDING.getStatus()); + confDO.setCreatedAt(createAt); + confDO.setCreatedBy("admin"); + confMapper.insert(confDO); + DictConfDO confDODb = confMapper.selectById(1L); + + confDO.setStatus(StatusEnum.OFFLINE.getStatus()); + // config.setMetricId(3L); + config.setBlackList(new ArrayList<>(Arrays.asList("p2"))); + config.setWhiteList(new ArrayList<>(Arrays.asList("p10", "p12"))); + confDODb.setConfig(JsonUtil.toString(config)); + confMapper.updateById(confDODb); + DictConfDO confDODb1 = confMapper.selectById(1L); + System.out.println(confDODb1.getId()); + } + + @Test + void testBatchInsert() { + for (int i = 0; i < 5; i++) { + insertConf(); + } + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.lambda().eq(DictConfDO::getType, "DIMENSION"); + List dictConfDOList = confMapper.selectList(wrapper); + System.out.println(dictConfDOList); + } + + @Test + void testAddTask() { + editConf(); + DictConfDO confDODb = confMapper.selectById(1L); + DictSingleTaskReq dictTask = DictSingleTaskReq.builder().itemId(confDODb.getItemId()) + .type(TypeEnums.DIMENSION).build(); + taskService.addDictTask(dictTask, null); + DictSingleTaskReq taskReq = DictSingleTaskReq.builder().itemId(3L).type(TypeEnums.DIMENSION).build(); + taskService.deleteDictTask(taskReq, null); + System.out.println(); + } + +} \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 220d9fe85..766e84531 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.util; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.core.agent.Agent; diff --git a/launchers/standalone/src/test/resources/META-INF/spring.factories b/launchers/standalone/src/test/resources/META-INF/spring.factories index 05086cb32..603198421 100644 --- a/launchers/standalone/src/test/resources/META-INF/spring.factories +++ b/launchers/standalone/src/test/resources/META-INF/spring.factories @@ -27,8 +27,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor, \ com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor -com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\ - com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter +com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\ + com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 787ebaefc..973d55d21 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -394,19 +394,30 @@ CREATE TABLE IF NOT EXISTS `singer` ( ); COMMENT ON TABLE singer IS 'singer_info'; +CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` ( + `id` INT NOT NULL AUTO_INCREMENT, + `description` varchar(255) , + `type` varchar(255) NOT NULL , + `item_id` INT NOT NULL , -- task Request Parameters md5 + `config` LONGVARCHAR , -- remark related information + `status` varchar(255) NOT NULL , -- the final status of the task + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `created_by` varchar(100) NOT NULL , + PRIMARY KEY (`id`) +); +COMMENT ON TABLE s2_dictionary_conf IS 'dictionary conf information table'; + CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( `id` INT NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL , -- task name `description` varchar(255) , - `command`LONGVARCHAR NOT NULL , -- task Request Parameters - `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 - `status` INT NOT NULL , -- the final status of the task - `dimension_ids` varchar(500) NULL , + `type` varchar(255) NOT NULL , + `item_id` INT NOT NULL , -- task Request Parameters md5 + `config` LONGVARCHAR , -- remark related information + `status` varchar(255) NOT NULL , -- the final status of the task `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `created_by` varchar(100) NOT NULL , - `progress` DOUBLE default 0.00 , -- task real-time progress `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds - `message` LONGVARCHAR , -- remark related information PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';