mirror of
https://github.com/tencentmusic/supersonic.git
synced 2026-04-18 20:34:19 +08:00
Compare commits
24 Commits
7490dabdc3
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0bbab5e7b1 | ||
|
|
d373567cef | ||
|
|
80a1f62d3f | ||
|
|
4379af4bd4 | ||
|
|
8a4bccab10 | ||
|
|
507af43b73 | ||
|
|
c1d50f978d | ||
|
|
18ce934bba | ||
|
|
6fe0ebcb9d | ||
|
|
77d8d63df7 | ||
|
|
0876f5eae8 | ||
|
|
ddbaf53ad4 | ||
|
|
4c97d01eab | ||
|
|
008f1443cb | ||
|
|
29c1119ee2 | ||
|
|
d658e437fb | ||
|
|
b6f561f18c | ||
|
|
593d26a072 | ||
|
|
9162b922c4 | ||
|
|
1d9324f689 | ||
|
|
6c5f8fce40 | ||
|
|
04b1edb2e2 | ||
|
|
9857256488 | ||
|
|
d695bed75d |
113
CLAUDE.md
Normal file
113
CLAUDE.md
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
# CLAUDE.md
|
||||||
|
|
||||||
|
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||||
|
|
||||||
|
## Build Commands
|
||||||
|
|
||||||
|
### Backend (Java/Maven)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Clean build (skip tests)
|
||||||
|
mvn clean package -DskipTests -Dspotless.skip=true
|
||||||
|
|
||||||
|
# Run all tests
|
||||||
|
mvn test
|
||||||
|
|
||||||
|
# Run single test class
|
||||||
|
mvn test -Dtest=ClassName
|
||||||
|
|
||||||
|
# Full CI build
|
||||||
|
mvn -B package --file pom.xml
|
||||||
|
```
|
||||||
|
|
||||||
|
**Requirements:** Java 21, Maven
|
||||||
|
|
||||||
|
### Frontend (pnpm/React)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd webapp
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
pnpm install
|
||||||
|
|
||||||
|
# Start dev server (port 9000)
|
||||||
|
pnpm dev
|
||||||
|
|
||||||
|
# Production build
|
||||||
|
pnpm build
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
pnpm test
|
||||||
|
```
|
||||||
|
|
||||||
|
**Requirements:** Node.js >=16, pnpm 9.12.3+
|
||||||
|
|
||||||
|
### Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build full release
|
||||||
|
./assembly/bin/supersonic-build.sh standalone
|
||||||
|
|
||||||
|
# Start service
|
||||||
|
./assembly/bin/supersonic-daemon.sh start
|
||||||
|
|
||||||
|
# Stop service
|
||||||
|
./assembly/bin/supersonic-daemon.sh stop
|
||||||
|
```
|
||||||
|
|
||||||
|
Visit http://localhost:9080 after startup.
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
SuperSonic unifies **Chat BI** (LLM-powered) and **Headless BI** (semantic layer) paradigms.
|
||||||
|
|
||||||
|
### Core Modules
|
||||||
|
|
||||||
|
```
|
||||||
|
supersonic/
|
||||||
|
├── auth/ # Authentication & authorization (SPI-based)
|
||||||
|
├── chat/ # Chat BI module - LLM-powered Q&A interface
|
||||||
|
├── common/ # Shared utilities
|
||||||
|
├── headless/ # Headless BI - semantic layer with open API
|
||||||
|
├── launchers/ # Application entry points
|
||||||
|
│ ├── standalone/ # Combined Chat + Headless (default)
|
||||||
|
│ ├── chat/ # Chat-only service
|
||||||
|
│ └── headless/ # Headless-only service
|
||||||
|
└── webapp/ # Frontend React app (UmiJS 4 + Ant Design)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Data Flow
|
||||||
|
|
||||||
|
1. **Knowledge Base**: Extracts schema from semantic models, builds dictionary/index for schema mapping
|
||||||
|
2. **Schema Mapper**: Identifies metrics/dimensions/entities/values in user queries
|
||||||
|
3. **Semantic Parser**: Generates S2SQL (semantic SQL) using rule-based and LLM-based parsers
|
||||||
|
4. **Semantic Corrector**: Validates and corrects semantic queries
|
||||||
|
5. **Semantic Translator**: Converts S2SQL to executable SQL
|
||||||
|
|
||||||
|
### Key Entry Points
|
||||||
|
|
||||||
|
- `StandaloneLauncher.java` - Combined service with `scanBasePackages: ["com.tencent.supersonic", "dev.langchain4j"]`
|
||||||
|
- `ChatLauncher.java` - Chat BI only
|
||||||
|
- `HeadlessLauncher.java` - Headless BI only
|
||||||
|
|
||||||
|
## Key Technologies
|
||||||
|
|
||||||
|
**Backend:** Spring Boot 3.3.9, MyBatis-Plus 3.5.10.1, LangChain4j 0.36.2, JSqlParser 4.9, Calcite 1.38.0
|
||||||
|
|
||||||
|
**Frontend:** React 18, UmiJS 4, Ant Design 5.17.4, ECharts 5.0.2, AntV G6/X6
|
||||||
|
|
||||||
|
**Databases:** MySQL, PostgreSQL (with pgvector), H2, ClickHouse, StarRocks, Presto, Trino, DuckDB
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
**Java tests:** JUnit 5, Mockito. Located in `src/test/java/` of each module.
|
||||||
|
|
||||||
|
**Frontend tests:** Jest with Puppeteer environment in `webapp/packages/supersonic-fe/`
|
||||||
|
|
||||||
|
**Evaluation scripts:** Python scripts in `evaluation/` directory for Text2SQL accuracy testing.
|
||||||
|
|
||||||
|
## Related Documentation
|
||||||
|
|
||||||
|
- [README.md](README.md) - English documentation
|
||||||
|
- [README_CN.md](README_CN.md) - Chinese documentation
|
||||||
|
- [Evaluation Guide](evaluation/README.md) - Text2SQL evaluation process
|
||||||
@@ -15,4 +15,6 @@ public class UserReq {
|
|||||||
|
|
||||||
@NotBlank(message = "password can not be null")
|
@NotBlank(message = "password can not be null")
|
||||||
private String newPassword;
|
private String newPassword;
|
||||||
|
|
||||||
|
private String role;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,4 +18,5 @@ public class ChatExecuteReq {
|
|||||||
private int parseId;
|
private int parseId;
|
||||||
private String queryText;
|
private String queryText;
|
||||||
private boolean saveAnswer;
|
private boolean saveAnswer;
|
||||||
|
private boolean streamingResult;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ public class QueryResp {
|
|||||||
private Long questionId;
|
private Long questionId;
|
||||||
private Date createTime;
|
private Date createTime;
|
||||||
private Long chatId;
|
private Long chatId;
|
||||||
|
private Integer agentId;
|
||||||
private Integer score;
|
private Integer score;
|
||||||
private String feedback;
|
private String feedback;
|
||||||
private String queryText;
|
private String queryText;
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ import dev.langchain4j.model.input.PromptTemplate;
|
|||||||
import dev.langchain4j.model.output.Response;
|
import dev.langchain4j.model.output.Response;
|
||||||
import dev.langchain4j.provider.ModelProvider;
|
import dev.langchain4j.provider.ModelProvider;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
@@ -171,10 +172,6 @@ public class NL2SQLParser implements ChatQueryParser {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// derive mapping result of current question and parsing result of last question.
|
|
||||||
ChatLayerService chatLayerService = ContextUtils.getBean(ChatLayerService.class);
|
|
||||||
MapResp currentMapResult = chatLayerService.map(queryNLReq);
|
|
||||||
|
|
||||||
List<QueryResp> historyQueries =
|
List<QueryResp> historyQueries =
|
||||||
getHistoryQueries(parseContext.getRequest().getChatId(), 1);
|
getHistoryQueries(parseContext.getRequest().getChatId(), 1);
|
||||||
if (historyQueries.isEmpty()) {
|
if (historyQueries.isEmpty()) {
|
||||||
@@ -182,12 +179,18 @@ public class NL2SQLParser implements ChatQueryParser {
|
|||||||
}
|
}
|
||||||
QueryResp lastQuery = historyQueries.get(0);
|
QueryResp lastQuery = historyQueries.get(0);
|
||||||
SemanticParseInfo lastParseInfo = lastQuery.getParseInfos().get(0);
|
SemanticParseInfo lastParseInfo = lastQuery.getParseInfos().get(0);
|
||||||
Long dataId = lastParseInfo.getDataSetId();
|
String histSQL = lastParseInfo.getSqlInfo().getCorrectedS2SQL();
|
||||||
|
if (StringUtils.isBlank(histSQL)) // 优化性能,如果问答不是chat bi 则无需重写,因为数据都不全
|
||||||
|
return;
|
||||||
|
|
||||||
|
// derive mapping result of current question and parsing result of last question.
|
||||||
|
ChatLayerService chatLayerService = ContextUtils.getBean(ChatLayerService.class);
|
||||||
|
MapResp currentMapResult = chatLayerService.map(queryNLReq); // 优化性能 ,只有满足条件才mapping
|
||||||
|
|
||||||
|
Long dataId = lastParseInfo.getDataSetId();
|
||||||
String curtMapStr =
|
String curtMapStr =
|
||||||
generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId));
|
generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId));
|
||||||
String histMapStr = generateSchemaPrompt(lastParseInfo.getElementMatches());
|
String histMapStr = generateSchemaPrompt(lastParseInfo.getElementMatches());
|
||||||
String histSQL = lastParseInfo.getSqlInfo().getCorrectedS2SQL();
|
|
||||||
|
|
||||||
Map<String, Object> variables = new HashMap<>();
|
Map<String, Object> variables = new HashMap<>();
|
||||||
variables.put("current_question", currentMapResult.getQueryText());
|
variables.put("current_question", currentMapResult.getQueryText());
|
||||||
|
|||||||
@@ -1,13 +1,20 @@
|
|||||||
package com.tencent.supersonic.chat.server.processor.execute;
|
package com.tencent.supersonic.chat.server.processor.execute;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSON;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
|
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
|
||||||
|
import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||||
import com.tencent.supersonic.common.pojo.ChatApp;
|
import com.tencent.supersonic.common.pojo.ChatApp;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AppModule;
|
import com.tencent.supersonic.common.pojo.enums.AppModule;
|
||||||
import com.tencent.supersonic.common.util.ChatAppManager;
|
import com.tencent.supersonic.common.util.ChatAppManager;
|
||||||
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import dev.langchain4j.data.message.AiMessage;
|
import dev.langchain4j.data.message.AiMessage;
|
||||||
|
import dev.langchain4j.model.StreamingResponseHandler;
|
||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
|
import dev.langchain4j.model.chat.StreamingChatLanguageModel;
|
||||||
import dev.langchain4j.model.input.Prompt;
|
import dev.langchain4j.model.input.Prompt;
|
||||||
import dev.langchain4j.model.input.PromptTemplate;
|
import dev.langchain4j.model.input.PromptTemplate;
|
||||||
import dev.langchain4j.model.output.Response;
|
import dev.langchain4j.model.output.Response;
|
||||||
@@ -24,9 +31,11 @@ import java.util.Objects;
|
|||||||
* DataInterpretProcessor interprets query result to make it more readable to the users.
|
* DataInterpretProcessor interprets query result to make it more readable to the users.
|
||||||
*/
|
*/
|
||||||
public class DataInterpretProcessor implements ExecuteResultProcessor {
|
public class DataInterpretProcessor implements ExecuteResultProcessor {
|
||||||
|
public static String tip = "AI 回答中...\r\n";
|
||||||
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
|
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
|
||||||
|
|
||||||
|
private static Map<Long, StringBuffer> resultCache = new HashMap<>();
|
||||||
|
|
||||||
public static final String APP_KEY = "DATA_INTERPRETER";
|
public static final String APP_KEY = "DATA_INTERPRETER";
|
||||||
private static final String INSTRUCTION = ""
|
private static final String INSTRUCTION = ""
|
||||||
+ "#Role: You are a data expert who communicates with business users everyday."
|
+ "#Role: You are a data expert who communicates with business users everyday."
|
||||||
@@ -41,6 +50,16 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
|
|||||||
.appModule(AppModule.CHAT).description("通过大模型对结果数据做提炼总结").enable(false).build());
|
.appModule(AppModule.CHAT).description("通过大模型对结果数据做提炼总结").enable(false).build());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String getTextSummary(Long queryId) {
|
||||||
|
if (resultCache.get(queryId) != null) {
|
||||||
|
return resultCache.get(queryId).toString();
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<Long, StringBuffer> getResultCache() {
|
||||||
|
return resultCache;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean accept(ExecuteContext executeContext) {
|
public boolean accept(ExecuteContext executeContext) {
|
||||||
@@ -71,14 +90,49 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
|
|||||||
variable.put("data", queryResult.getTextResult());
|
variable.put("data", queryResult.getTextResult());
|
||||||
|
|
||||||
Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variable);
|
Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variable);
|
||||||
ChatLanguageModel chatLanguageModel =
|
if (executeContext.getRequest().isStreamingResult()) {
|
||||||
ModelProvider.getChatModel(chatApp.getChatModelConfig());
|
StreamingChatLanguageModel chatLanguageModel =
|
||||||
Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
|
ModelProvider.getChatStreamingModel(chatApp.getChatModelConfig());
|
||||||
String anwser = response.content().text();
|
final Long queryId = executeContext.getRequest().getQueryId();
|
||||||
keyPipelineLog.info("DataInterpretProcessor modelReq:\n{} \nmodelResp:\n{}", prompt.text(),
|
resultCache.put(queryId, new StringBuffer(tip));
|
||||||
anwser);
|
chatLanguageModel.generate(prompt.toUserMessage(),
|
||||||
if (StringUtils.isNotBlank(anwser)) {
|
new StreamingResponseHandler<AiMessage>() {
|
||||||
queryResult.setTextSummary(anwser);
|
@Override
|
||||||
|
public void onNext(String token) {
|
||||||
|
resultCache.get(queryId).append(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onComplete(Response<AiMessage> response) {
|
||||||
|
ChatQueryRepository chatQueryRepository =
|
||||||
|
ContextUtils.getBean(ChatQueryRepository.class);
|
||||||
|
ChatQueryDO chatQueryDO = chatQueryRepository.getChatQueryDO(queryId);
|
||||||
|
JSONObject queryResult = JSON.parseObject(chatQueryDO.getQueryResult());
|
||||||
|
queryResult.put("textSummary",
|
||||||
|
resultCache.get(queryId).toString().substring(tip.length()));
|
||||||
|
chatQueryDO.setQueryResult(queryResult.toJSONString());
|
||||||
|
chatQueryRepository.updateChatQuery(chatQueryDO);
|
||||||
|
resultCache.remove(queryId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onError(Throwable error) {
|
||||||
|
error.printStackTrace();
|
||||||
|
resultCache.remove(queryId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
ChatLanguageModel chatLanguageModel =
|
||||||
|
ModelProvider.getChatModel(chatApp.getChatModelConfig());
|
||||||
|
Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
|
||||||
|
String anwser = response.content().text();
|
||||||
|
keyPipelineLog.info("DataInterpretProcessor modelReq:\n{} \nmodelResp:\n{}",
|
||||||
|
prompt.text(), anwser);
|
||||||
|
if (StringUtils.isNotBlank(anwser)) {
|
||||||
|
queryResult.setTextSummary(anwser);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
|||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.ChatParseResp;
|
import com.tencent.supersonic.chat.api.pojo.response.ChatParseResp;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
||||||
import com.tencent.supersonic.common.pojo.User;
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||||
@@ -50,6 +51,14 @@ public class ChatQueryController {
|
|||||||
return chatQueryService.execute(chatExecuteReq);
|
return chatQueryService.execute(chatExecuteReq);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("getExecuteSummary")
|
||||||
|
public Object getExecuteSummary(@RequestBody ChatExecuteReq chatExecuteReq,
|
||||||
|
HttpServletRequest request, HttpServletResponse response) {
|
||||||
|
chatExecuteReq.setUser(UserHolder.findUser(request, response));
|
||||||
|
QueryResult res = chatQueryService.getTextSummary(chatExecuteReq);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/")
|
@PostMapping("/")
|
||||||
public Object query(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
|
public Object query(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
|
||||||
HttpServletResponse response) throws Exception {
|
HttpServletResponse response) throws Exception {
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ public interface ChatManageService {
|
|||||||
|
|
||||||
QueryResp getChatQuery(Long queryId);
|
QueryResp getChatQuery(Long queryId);
|
||||||
|
|
||||||
|
ChatQueryDO getChatQueryDO(Long queryId);
|
||||||
|
|
||||||
List<QueryResp> getChatQueries(Integer chatId);
|
List<QueryResp> getChatQueries(Integer chatId);
|
||||||
|
|
||||||
ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId);
|
ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId);
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ public interface ChatQueryService {
|
|||||||
|
|
||||||
QueryResult execute(ChatExecuteReq chatExecuteReq) throws Exception;
|
QueryResult execute(ChatExecuteReq chatExecuteReq) throws Exception;
|
||||||
|
|
||||||
|
QueryResult getTextSummary(ChatExecuteReq chatExecuteReq);
|
||||||
|
|
||||||
QueryResult parseAndExecute(ChatParseReq chatParseReq);
|
QueryResult parseAndExecute(ChatParseReq chatParseReq);
|
||||||
|
|
||||||
Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception;
|
Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception;
|
||||||
|
|||||||
@@ -123,6 +123,11 @@ public class ChatManageServiceImpl implements ChatManageService {
|
|||||||
return chatQueryRepository.getChatQuery(queryId);
|
return chatQueryRepository.getChatQuery(queryId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChatQueryDO getChatQueryDO(Long queryId) {
|
||||||
|
return chatQueryRepository.getChatQueryDO(queryId);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<QueryResp> getChatQueries(Integer chatId) {
|
public List<QueryResp> getChatQueries(Integer chatId) {
|
||||||
List<QueryResp> queries = chatQueryRepository.getChatQueries(chatId);
|
List<QueryResp> queries = chatQueryRepository.getChatQueries(chatId);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.tencent.supersonic.chat.server.service.impl;
|
package com.tencent.supersonic.chat.server.service.impl;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||||
@@ -9,8 +10,10 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
|||||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
import com.tencent.supersonic.chat.server.executor.ChatQueryExecutor;
|
import com.tencent.supersonic.chat.server.executor.ChatQueryExecutor;
|
||||||
import com.tencent.supersonic.chat.server.parser.ChatQueryParser;
|
import com.tencent.supersonic.chat.server.parser.ChatQueryParser;
|
||||||
|
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ParseContext;
|
import com.tencent.supersonic.chat.server.pojo.ParseContext;
|
||||||
|
import com.tencent.supersonic.chat.server.processor.execute.DataInterpretProcessor;
|
||||||
import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;
|
import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;
|
||||||
import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
|
import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
|
||||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||||
@@ -18,7 +21,11 @@ import com.tencent.supersonic.chat.server.service.ChatManageService;
|
|||||||
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
||||||
import com.tencent.supersonic.chat.server.util.ComponentFactory;
|
import com.tencent.supersonic.chat.server.util.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
|
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
|
||||||
import com.tencent.supersonic.common.jsqlparser.*;
|
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
|
||||||
|
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
|
||||||
|
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
|
||||||
|
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
|
||||||
|
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
|
||||||
import com.tencent.supersonic.common.pojo.User;
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||||
import com.tencent.supersonic.common.util.DateUtils;
|
import com.tencent.supersonic.common.util.DateUtils;
|
||||||
@@ -44,7 +51,11 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import net.sf.jsqlparser.expression.Expression;
|
import net.sf.jsqlparser.expression.Expression;
|
||||||
import net.sf.jsqlparser.expression.LongValue;
|
import net.sf.jsqlparser.expression.LongValue;
|
||||||
import net.sf.jsqlparser.expression.StringValue;
|
import net.sf.jsqlparser.expression.StringValue;
|
||||||
import net.sf.jsqlparser.expression.operators.relational.*;
|
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.InExpression;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
|
||||||
|
import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
|
||||||
import net.sf.jsqlparser.schema.Column;
|
import net.sf.jsqlparser.schema.Column;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
@@ -53,7 +64,14 @@ import org.springframework.context.annotation.Lazy;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@@ -110,6 +128,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!parseContext.needFeedback()) {
|
if (!parseContext.needFeedback()) {
|
||||||
|
parseContext.getResponse().getParseTimeCost().setParseTime(System.currentTimeMillis()
|
||||||
|
- parseContext.getResponse().getParseTimeCost().getParseStartTime());
|
||||||
chatManageService.batchAddParse(chatParseReq, parseContext.getResponse());
|
chatManageService.batchAddParse(chatParseReq, parseContext.getResponse());
|
||||||
chatManageService.updateParseCostTime(parseContext.getResponse());
|
chatManageService.updateParseCostTime(parseContext.getResponse());
|
||||||
}
|
}
|
||||||
@@ -143,6 +163,21 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
return queryResult;
|
return queryResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResult getTextSummary(ChatExecuteReq chatExecuteReq) {
|
||||||
|
String text = DataInterpretProcessor.getTextSummary(chatExecuteReq.getQueryId());
|
||||||
|
if (StringUtils.isNotBlank(text)) {
|
||||||
|
QueryResult res = new QueryResult();
|
||||||
|
res.setTextSummary(text);
|
||||||
|
res.setQueryId(chatExecuteReq.getQueryId());
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
ChatQueryDO chatQueryDo = chatManageService.getChatQueryDO(chatExecuteReq.getQueryId());
|
||||||
|
QueryResult res = JSON.parseObject(chatQueryDo.getQueryResult(), QueryResult.class);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResult parseAndExecute(ChatParseReq chatParseReq) {
|
public QueryResult parseAndExecute(ChatParseReq chatParseReq) {
|
||||||
ChatParseResp parseResp = parse(chatParseReq);
|
ChatParseResp parseResp = parse(chatParseReq);
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ public class PluginServiceImpl implements PluginService {
|
|||||||
if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) {
|
if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) {
|
||||||
queryWrapper.lambda().eq(PluginDO::getCreatedBy, pluginQueryReq.getCreatedBy());
|
queryWrapper.lambda().eq(PluginDO::getCreatedBy, pluginQueryReq.getCreatedBy());
|
||||||
}
|
}
|
||||||
|
queryWrapper.orderByAsc("name");
|
||||||
List<PluginDO> pluginDOS = pluginRepository.query(queryWrapper);
|
List<PluginDO> pluginDOS = pluginRepository.query(queryWrapper);
|
||||||
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
|
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
|
||||||
pluginDOS = pluginDOS.stream()
|
pluginDOS = pluginDOS.stream()
|
||||||
|
|||||||
@@ -118,22 +118,26 @@ public class SqlReplaceHelper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static void getFromSelect(FromItem fromItem, List<PlainSelect> plainSelectList) {
|
public static void getFromSelect(FromItem fromItem, List<PlainSelect> plainSelectList) {
|
||||||
if (!(fromItem instanceof ParenthesedSelect)) {
|
if (!(fromItem instanceof ParenthesedSelect parenthesedSelect)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) fromItem;
|
|
||||||
Select select = parenthesedSelect.getSelect();
|
Select select = parenthesedSelect.getSelect();
|
||||||
if (select instanceof PlainSelect) {
|
if (select instanceof PlainSelect plainSelect) {
|
||||||
PlainSelect plainSelect = (PlainSelect) select;
|
|
||||||
plainSelectList.add(plainSelect);
|
plainSelectList.add(plainSelect);
|
||||||
getFromSelect(plainSelect.getFromItem(), plainSelectList);
|
getFromSelect(plainSelect.getFromItem(), plainSelectList);
|
||||||
} else if (select instanceof SetOperationList) {
|
} else if (select instanceof SetOperationList setOperationList) {
|
||||||
SetOperationList setOperationList = (SetOperationList) select;
|
|
||||||
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
|
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
|
||||||
setOperationList.getSelects().forEach(subSelectBody -> {
|
setOperationList.getSelects().forEach(subSelectBody -> {
|
||||||
PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
|
if (subSelectBody instanceof PlainSelect subPlainSelect) {
|
||||||
plainSelectList.add(subPlainSelect);
|
plainSelectList.add(subPlainSelect);
|
||||||
getFromSelect(subPlainSelect.getFromItem(), plainSelectList);
|
getFromSelect(subPlainSelect.getFromItem(), plainSelectList);
|
||||||
|
} else if (subSelectBody instanceof ParenthesedSelect subParenthesedSelect) {
|
||||||
|
Select innerSelect = subParenthesedSelect.getSelect();
|
||||||
|
if (innerSelect instanceof PlainSelect innerPlainSelect) {
|
||||||
|
plainSelectList.add(innerPlainSelect);
|
||||||
|
getFromSelect(innerPlainSelect.getFromItem(), plainSelectList);
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -188,8 +192,13 @@ public class SqlReplaceHelper {
|
|||||||
SetOperationList setOperationList = (SetOperationList) select;
|
SetOperationList setOperationList = (SetOperationList) select;
|
||||||
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
|
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
|
||||||
setOperationList.getSelects().forEach(subSelectBody -> {
|
setOperationList.getSelects().forEach(subSelectBody -> {
|
||||||
PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
|
if (subSelectBody instanceof PlainSelect) {
|
||||||
replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect);
|
PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
|
||||||
|
replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect);
|
||||||
|
} else if (subSelectBody instanceof ParenthesedSelect) {
|
||||||
|
replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace,
|
||||||
|
((ParenthesedSelect) subSelectBody).getPlainSelect());
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,27 +1,26 @@
|
|||||||
package com.tencent.supersonic.common.pojo;
|
package com.tencent.supersonic.common.pojo;
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||||
|
import lombok.Getter;
|
||||||
import org.springframework.context.ApplicationEvent;
|
import org.springframework.context.ApplicationEvent;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
@Getter
|
||||||
public class DataEvent extends ApplicationEvent {
|
public class DataEvent extends ApplicationEvent {
|
||||||
|
|
||||||
private List<DataItem> dataItems;
|
private final List<DataItem> dataItems;
|
||||||
|
|
||||||
private EventType eventType;
|
private final EventType eventType;
|
||||||
|
|
||||||
public DataEvent(Object source, List<DataItem> dataItems, EventType eventType) {
|
private final String userName;
|
||||||
|
|
||||||
|
public DataEvent(Object source, List<DataItem> dataItems, EventType eventType,
|
||||||
|
String userName) {
|
||||||
super(source);
|
super(source);
|
||||||
this.dataItems = dataItems;
|
this.dataItems = dataItems;
|
||||||
this.eventType = eventType;
|
this.eventType = eventType;
|
||||||
|
this.userName = userName;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<DataItem> getDataItems() {
|
|
||||||
return dataItems;
|
|
||||||
}
|
|
||||||
|
|
||||||
public EventType getEventType() {
|
|
||||||
return eventType;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,5 +15,5 @@ public interface ChatModelService {
|
|||||||
|
|
||||||
ChatModel updateChatModel(ChatModel chatModel, User user);
|
ChatModel updateChatModel(ChatModel chatModel, User user);
|
||||||
|
|
||||||
void deleteChatModel(Integer id);
|
void deleteChatModel(Integer id, User user);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,7 +79,12 @@ public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModel
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteChatModel(Integer id) {
|
public void deleteChatModel(Integer id, User user) {
|
||||||
|
ChatModel chatModel = getChatModel(id);
|
||||||
|
if (!checkAdminPermission(user, chatModel)) {
|
||||||
|
throw new RuntimeException("没有权限删除该大模型");
|
||||||
|
}
|
||||||
|
|
||||||
removeById(id);
|
removeById(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -103,4 +108,13 @@ public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModel
|
|||||||
chatModelDO.setConfig(JsonUtil.toString(chatModel.getConfig()));
|
chatModelDO.setConfig(JsonUtil.toString(chatModel.getConfig()));
|
||||||
return chatModelDO;
|
return chatModelDO;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean checkAdminPermission(User user, ChatModel chatModel) {
|
||||||
|
String admin = chatModel.getAdmin();
|
||||||
|
if (user.isSuperAdmin()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return admin != null && admin.equals(user.getName())
|
||||||
|
|| chatModel.getCreatedBy().equals(user.getName());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import org.springframework.util.CollectionUtils;
|
|||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class SystemConfigServiceImpl extends ServiceImpl<SystemConfigMapper, SystemConfigDO>
|
public class SystemConfigServiceImpl extends ServiceImpl<SystemConfigMapper, SystemConfigDO>
|
||||||
@@ -38,8 +39,8 @@ public class SystemConfigServiceImpl extends ServiceImpl<SystemConfigMapper, Sys
|
|||||||
return systemConfigDb;
|
return systemConfigDb;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SystemConfig getSystemConfigFromDB() {
|
private SystemConfig getSystemConfigFromDB() { // 加上id ,如果有多条记录,会出错
|
||||||
List<SystemConfigDO> list = list();
|
List<SystemConfigDO> list = this.lambdaQuery().eq(SystemConfigDO::getId, 1).list();
|
||||||
if (CollectionUtils.isEmpty(list)) {
|
if (CollectionUtils.isEmpty(list)) {
|
||||||
SystemConfig systemConfig = new SystemConfig();
|
SystemConfig systemConfig = new SystemConfig();
|
||||||
systemConfig.setId(1);
|
systemConfig.setId(1);
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import dev.langchain4j.model.chat.ChatLanguageModel;
|
|||||||
import dev.langchain4j.model.dify.DifyAiChatModel;
|
import dev.langchain4j.model.dify.DifyAiChatModel;
|
||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import dev.langchain4j.model.openai.OpenAiEmbeddingModel;
|
import dev.langchain4j.model.openai.OpenAiEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
import org.springframework.beans.factory.InitializingBean;
|
import org.springframework.beans.factory.InitializingBean;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -25,6 +26,11 @@ public class DifyModelFactory implements ModelFactory, InitializingBean {
|
|||||||
.modelName(modelConfig.getModelName()).timeOut(modelConfig.getTimeOut()).build();
|
.modelName(modelConfig.getModelName()).timeOut(modelConfig.getTimeOut()).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public OpenAiStreamingChatModel createChatStreamingModel(ChatModelConfig modelConfig) {
|
||||||
|
throw new RuntimeException("待开发");
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModelConfig) {
|
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModelConfig) {
|
||||||
return OpenAiEmbeddingModel.builder().baseUrl(embeddingModelConfig.getBaseUrl())
|
return OpenAiEmbeddingModel.builder().baseUrl(embeddingModelConfig.getBaseUrl())
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
|||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import dev.langchain4j.model.embedding.S2OnnxEmbeddingModel;
|
import dev.langchain4j.model.embedding.S2OnnxEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.InitializingBean;
|
import org.springframework.beans.factory.InitializingBean;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -35,6 +36,11 @@ public class InMemoryModelFactory implements ModelFactory, InitializingBean {
|
|||||||
return EmbeddingModelConstant.BGE_SMALL_ZH_MODEL;
|
return EmbeddingModelConstant.BGE_SMALL_ZH_MODEL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public OpenAiStreamingChatModel createChatStreamingModel(ChatModelConfig modelConfig) {
|
||||||
|
throw new RuntimeException("待开发");
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void afterPropertiesSet() {
|
public void afterPropertiesSet() {
|
||||||
ModelProvider.add(PROVIDER, this);
|
ModelProvider.add(PROVIDER, this);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import dev.langchain4j.model.chat.ChatLanguageModel;
|
|||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import dev.langchain4j.model.localai.LocalAiChatModel;
|
import dev.langchain4j.model.localai.LocalAiChatModel;
|
||||||
import dev.langchain4j.model.localai.LocalAiEmbeddingModel;
|
import dev.langchain4j.model.localai.LocalAiEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
import org.springframework.beans.factory.InitializingBean;
|
import org.springframework.beans.factory.InitializingBean;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -27,6 +28,11 @@ public class LocalAiModelFactory implements ModelFactory, InitializingBean {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public OpenAiStreamingChatModel createChatStreamingModel(ChatModelConfig modelConfig) {
|
||||||
|
throw new RuntimeException("待开发");
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModel) {
|
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModel) {
|
||||||
return LocalAiEmbeddingModel.builder().baseUrl(embeddingModel.getBaseUrl())
|
return LocalAiEmbeddingModel.builder().baseUrl(embeddingModel.getBaseUrl())
|
||||||
|
|||||||
@@ -4,9 +4,12 @@ import com.tencent.supersonic.common.pojo.ChatModelConfig;
|
|||||||
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
|
|
||||||
public interface ModelFactory {
|
public interface ModelFactory {
|
||||||
ChatLanguageModel createChatModel(ChatModelConfig modelConfig);
|
ChatLanguageModel createChatModel(ChatModelConfig modelConfig);
|
||||||
|
|
||||||
|
OpenAiStreamingChatModel createChatStreamingModel(ChatModelConfig modelConfig);
|
||||||
|
|
||||||
EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModel);
|
EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModel);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ import com.tencent.supersonic.common.pojo.ChatModelConfig;
|
|||||||
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
|
import dev.langchain4j.model.chat.StreamingChatLanguageModel;
|
||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -41,6 +43,20 @@ public class ModelProvider {
|
|||||||
"Unsupported ChatLanguageModel provider: " + modelConfig.getProvider());
|
"Unsupported ChatLanguageModel provider: " + modelConfig.getProvider());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static StreamingChatLanguageModel getChatStreamingModel(ChatModelConfig modelConfig) {
|
||||||
|
if (modelConfig == null || StringUtils.isBlank(modelConfig.getProvider())
|
||||||
|
|| StringUtils.isBlank(modelConfig.getBaseUrl())) {
|
||||||
|
modelConfig = DEMO_CHAT_MODEL;
|
||||||
|
}
|
||||||
|
ModelFactory modelFactory = factories.get(modelConfig.getProvider().toUpperCase());
|
||||||
|
if (modelFactory != null) {
|
||||||
|
return modelFactory.createChatStreamingModel(modelConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new RuntimeException(
|
||||||
|
"Unsupported ChatLanguageModel provider: " + modelConfig.getProvider());
|
||||||
|
}
|
||||||
|
|
||||||
public static EmbeddingModel getEmbeddingModel() {
|
public static EmbeddingModel getEmbeddingModel() {
|
||||||
return getEmbeddingModel(null);
|
return getEmbeddingModel(null);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import dev.langchain4j.model.chat.ChatLanguageModel;
|
|||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import dev.langchain4j.model.ollama.OllamaChatModel;
|
import dev.langchain4j.model.ollama.OllamaChatModel;
|
||||||
import dev.langchain4j.model.ollama.OllamaEmbeddingModel;
|
import dev.langchain4j.model.ollama.OllamaEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
import org.springframework.beans.factory.InitializingBean;
|
import org.springframework.beans.factory.InitializingBean;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -28,6 +29,11 @@ public class OllamaModelFactory implements ModelFactory, InitializingBean {
|
|||||||
.logResponses(modelConfig.getLogResponses()).build();
|
.logResponses(modelConfig.getLogResponses()).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public OpenAiStreamingChatModel createChatStreamingModel(ChatModelConfig modelConfig) {
|
||||||
|
throw new RuntimeException("待开发");
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModelConfig) {
|
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModelConfig) {
|
||||||
return OllamaEmbeddingModel.builder().baseUrl(embeddingModelConfig.getBaseUrl())
|
return OllamaEmbeddingModel.builder().baseUrl(embeddingModelConfig.getBaseUrl())
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import dev.langchain4j.model.chat.ChatLanguageModel;
|
|||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import dev.langchain4j.model.openai.OpenAiChatModel;
|
import dev.langchain4j.model.openai.OpenAiChatModel;
|
||||||
import dev.langchain4j.model.openai.OpenAiEmbeddingModel;
|
import dev.langchain4j.model.openai.OpenAiEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;
|
||||||
import org.springframework.beans.factory.InitializingBean;
|
import org.springframework.beans.factory.InitializingBean;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -37,6 +38,16 @@ public class OpenAiModelFactory implements ModelFactory, InitializingBean {
|
|||||||
return openAiChatModelBuilder.build();
|
return openAiChatModelBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public OpenAiStreamingChatModel createChatStreamingModel(ChatModelConfig modelConfig) {
|
||||||
|
return OpenAiStreamingChatModel.builder().baseUrl(modelConfig.getBaseUrl())
|
||||||
|
.modelName(modelConfig.getModelName()).apiKey(modelConfig.keyDecrypt())
|
||||||
|
.temperature(modelConfig.getTemperature()).topP(modelConfig.getTopP())
|
||||||
|
.timeout(Duration.ofSeconds(modelConfig.getTimeOut()))
|
||||||
|
.logRequests(modelConfig.getLogRequests())
|
||||||
|
.logResponses(modelConfig.getLogResponses()).build();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModel) {
|
public EmbeddingModel createEmbeddingModel(EmbeddingModelConfig embeddingModel) {
|
||||||
return OpenAiEmbeddingModel.builder().baseUrl(embeddingModel.getBaseUrl())
|
return OpenAiEmbeddingModel.builder().baseUrl(embeddingModel.getBaseUrl())
|
||||||
|
|||||||
@@ -42,6 +42,6 @@ public class TextSegmentConvert {
|
|||||||
if (Objects.isNull(textSegment) || Objects.isNull(textSegment.metadata())) {
|
if (Objects.isNull(textSegment) || Objects.isNull(textSegment.metadata())) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return textSegment.metadata().get(QUERY_ID);
|
return textSegment.metadata().getString(QUERY_ID);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,4 +12,6 @@ public class DictSingleTaskReq {
|
|||||||
private TypeEnums type;
|
private TypeEnums type;
|
||||||
@NotNull
|
@NotNull
|
||||||
private Long itemId;
|
private Long itemId;
|
||||||
|
private String startDate;
|
||||||
|
private String endDate;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
|||||||
import jakarta.validation.constraints.NotNull;
|
import jakarta.validation.constraints.NotNull;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author: kanedai
|
* @author: kanedai
|
||||||
* @date: 2024/10/31
|
* @date: 2024/10/31
|
||||||
@@ -15,7 +17,7 @@ public class DimValueAliasReq {
|
|||||||
private Long id;
|
private Long id;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* alias为空代表删除 否则更新
|
* alias 为空代表删除 否则更新
|
||||||
*/
|
*/
|
||||||
DimValueMap dimValueMaps;
|
private List<DimValueMap> dimValueMaps;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ public class PageSchemaItemReq extends PageBaseReq {
|
|||||||
private String createdBy;
|
private String createdBy;
|
||||||
private List<Long> domainIds = Lists.newArrayList();
|
private List<Long> domainIds = Lists.newArrayList();
|
||||||
private List<Long> modelIds = Lists.newArrayList();
|
private List<Long> modelIds = Lists.newArrayList();
|
||||||
|
private Long dataSetId;
|
||||||
private Integer sensitiveLevel;
|
private Integer sensitiveLevel;
|
||||||
private Integer status;
|
private Integer status;
|
||||||
private String key;
|
private String key;
|
||||||
|
|||||||
@@ -8,43 +8,107 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class KnowledgeBaseService {
|
public class KnowledgeBaseService {
|
||||||
private static volatile Map<Long, List<DictWord>> dimValueAliasMap = new HashMap<>();
|
private static final Map<Long, List<DictWord>> dimValueAliasMap = new ConcurrentHashMap<>();
|
||||||
|
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get dimension value alias map (read-only).
|
||||||
|
*
|
||||||
|
* @return unmodifiable view of the map
|
||||||
|
*/
|
||||||
public static Map<Long, List<DictWord>> getDimValueAlias() {
|
public static Map<Long, List<DictWord>> getDimValueAlias() {
|
||||||
return dimValueAliasMap;
|
return Collections.unmodifiableMap(dimValueAliasMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add dimension value aliases with deduplication. Thread-safe implementation using
|
||||||
|
* ConcurrentHashMap.
|
||||||
|
*
|
||||||
|
* @param dimId dimension ID
|
||||||
|
* @param newWords new words to add
|
||||||
|
* @return updated list of aliases for the dimension
|
||||||
|
*/
|
||||||
public static List<DictWord> addDimValueAlias(Long dimId, List<DictWord> newWords) {
|
public static List<DictWord> addDimValueAlias(Long dimId, List<DictWord> newWords) {
|
||||||
List<DictWord> dimValueAlias =
|
if (dimId == null || CollectionUtils.isEmpty(newWords)) {
|
||||||
dimValueAliasMap.containsKey(dimId) ? dimValueAliasMap.get(dimId)
|
return dimValueAliasMap.get(dimId);
|
||||||
: new ArrayList<>();
|
}
|
||||||
Set<String> wordSet =
|
|
||||||
dimValueAlias
|
// Use computeIfAbsent and synchronized block for thread safety
|
||||||
.stream().map(word -> String.format("%s_%s_%s",
|
synchronized (dimValueAliasMap) {
|
||||||
word.getNatureWithFrequency(), word.getWord(), word.getAlias()))
|
List<DictWord> dimValueAlias =
|
||||||
.collect(Collectors.toSet());
|
dimValueAliasMap.computeIfAbsent(dimId, k -> new ArrayList<>());
|
||||||
for (DictWord dictWord : newWords) {
|
|
||||||
String key = String.format("%s_%s_%s", dictWord.getNatureWithFrequency(),
|
// Build deduplication key set
|
||||||
dictWord.getWord(), dictWord.getAlias());
|
Set<String> existingKeys = dimValueAlias.stream().map(word -> buildDedupKey(word))
|
||||||
if (!wordSet.contains(key)) {
|
.collect(Collectors.toSet());
|
||||||
dimValueAlias.add(dictWord);
|
|
||||||
}
|
// Add new words with deduplication
|
||||||
|
for (DictWord dictWord : newWords) {
|
||||||
|
String key = buildDedupKey(dictWord);
|
||||||
|
if (!existingKeys.contains(key)) {
|
||||||
|
dimValueAlias.add(dictWord);
|
||||||
|
existingKeys.add(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dimValueAlias;
|
||||||
}
|
}
|
||||||
dimValueAliasMap.put(dimId, dimValueAlias);
|
|
||||||
return dimValueAlias;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void updateSemanticKnowledge(List<DictWord> natures) {
|
/**
|
||||||
|
* Remove dimension value aliases by dimension ID.
|
||||||
|
*
|
||||||
|
* @param dimId dimension ID to remove, or null to clear all
|
||||||
|
*/
|
||||||
|
public static void removeDimValueAlias(Long dimId) {
|
||||||
|
if (dimId == null) {
|
||||||
|
dimValueAliasMap.clear();
|
||||||
|
log.info("Cleared all dimension value aliases");
|
||||||
|
} else {
|
||||||
|
dimValueAliasMap.remove(dimId);
|
||||||
|
log.info("Removed dimension value alias for dimId: {}", dimId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build deduplication key for DictWord.
|
||||||
|
*
|
||||||
|
* @param word the DictWord object
|
||||||
|
* @return deduplication key string
|
||||||
|
*/
|
||||||
|
private static String buildDedupKey(DictWord word) {
|
||||||
|
return String.format("%s_%s_%s", word.getNatureWithFrequency(), word.getWord(),
|
||||||
|
word.getAlias());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update semantic knowledge (incremental add, no clearing). Use this method to add new words
|
||||||
|
* without removing existing data.
|
||||||
|
*
|
||||||
|
* @param natures the words to add
|
||||||
|
*/
|
||||||
|
public void updateSemanticKnowledge(List<DictWord> natures) {
|
||||||
|
lock.writeLock().lock();
|
||||||
|
try {
|
||||||
|
updateSemanticKnowledgeInternal(natures);
|
||||||
|
} finally {
|
||||||
|
lock.writeLock().unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateSemanticKnowledgeInternal(List<DictWord> natures) {
|
||||||
List<DictWord> prefixes = natures.stream().filter(
|
List<DictWord> prefixes = natures.stream().filter(
|
||||||
entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
|
entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
@@ -60,52 +124,82 @@ public class KnowledgeBaseService {
|
|||||||
SearchService.loadSuffix(suffixes);
|
SearchService.loadSuffix(suffixes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reload all knowledge (full replacement with clearing). Use this method to rebuild the entire
|
||||||
|
* knowledge base.
|
||||||
|
*
|
||||||
|
* @param natures all words to load
|
||||||
|
*/
|
||||||
public void reloadAllData(List<DictWord> natures) {
|
public void reloadAllData(List<DictWord> natures) {
|
||||||
// 1. reload custom knowledge
|
// 1. reload custom knowledge (executed outside lock to avoid long blocking)
|
||||||
try {
|
try {
|
||||||
HanlpHelper.reloadCustomDictionary();
|
HanlpHelper.reloadCustomDictionary();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("reloadCustomDictionary error", e);
|
log.error("reloadCustomDictionary error", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. update online knowledge
|
// 2. acquire write lock, clear trie and rebuild (short operation)
|
||||||
if (CollectionUtils.isNotEmpty(dimValueAliasMap)) {
|
lock.writeLock().lock();
|
||||||
for (Long dimId : dimValueAliasMap.keySet()) {
|
|
||||||
natures.addAll(dimValueAliasMap.get(dimId));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
updateOnlineKnowledge(natures);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void updateOnlineKnowledge(List<DictWord> natures) {
|
|
||||||
try {
|
try {
|
||||||
updateSemanticKnowledge(natures);
|
SearchService.clear();
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("updateSemanticKnowledge error", e);
|
if (CollectionUtils.isNotEmpty(dimValueAliasMap)) {
|
||||||
|
for (Long dimId : dimValueAliasMap.keySet()) {
|
||||||
|
natures.addAll(dimValueAliasMap.get(dimId));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
updateSemanticKnowledgeInternal(natures);
|
||||||
|
} finally {
|
||||||
|
lock.writeLock().unlock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<S2Term> getTerms(String text, Map<Long, List<Long>> modelIdToDataSetIds) {
|
public List<S2Term> getTerms(String text, Map<Long, List<Long>> modelIdToDataSetIds) {
|
||||||
return HanlpHelper.getTerms(text, modelIdToDataSetIds);
|
lock.readLock().lock();
|
||||||
|
try {
|
||||||
|
return HanlpHelper.getTerms(text, modelIdToDataSetIds);
|
||||||
|
} finally {
|
||||||
|
lock.readLock().unlock();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<HanlpMapResult> prefixSearch(String key, int limit,
|
public List<HanlpMapResult> prefixSearch(String key, int limit,
|
||||||
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
||||||
return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
lock.readLock().lock();
|
||||||
|
try {
|
||||||
|
return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
||||||
|
} finally {
|
||||||
|
lock.readLock().unlock();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
|
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
|
||||||
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
||||||
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
lock.readLock().lock();
|
||||||
|
try {
|
||||||
|
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
||||||
|
} finally {
|
||||||
|
lock.readLock().unlock();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<HanlpMapResult> suffixSearch(String key, int limit,
|
public List<HanlpMapResult> suffixSearch(String key, int limit,
|
||||||
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
||||||
return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
lock.readLock().lock();
|
||||||
|
try {
|
||||||
|
return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
||||||
|
} finally {
|
||||||
|
lock.readLock().unlock();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<HanlpMapResult> suffixSearchByModel(String key, int limit,
|
public List<HanlpMapResult> suffixSearchByModel(String key, int limit,
|
||||||
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
|
||||||
return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
lock.readLock().lock();
|
||||||
|
try {
|
||||||
|
return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
|
||||||
|
} finally {
|
||||||
|
lock.readLock().unlock();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,9 +117,17 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
|
|||||||
dictWord.setAlias(word.toLowerCase());
|
dictWord.setAlias(word.toLowerCase());
|
||||||
String[] split = nature.split(DictWordType.NATURE_SPILT);
|
String[] split = nature.split(DictWordType.NATURE_SPILT);
|
||||||
if (split.length >= 2) {
|
if (split.length >= 2) {
|
||||||
Long dimId = Long.parseLong(
|
try {
|
||||||
nature.split(DictWordType.NATURE_SPILT)[split.length - 1]);
|
Long dimId = Long.parseLong(
|
||||||
KnowledgeBaseService.addDimValueAlias(dimId, Arrays.asList(dictWord));
|
nature.split(DictWordType.NATURE_SPILT)[split.length - 1]);
|
||||||
|
KnowledgeBaseService.addDimValueAlias(dimId,
|
||||||
|
Arrays.asList(dictWord));
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
logger.warning(path + " : 非标准文件,不存入KnowledgeBaseService");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ import java.util.stream.Collectors;
|
|||||||
public class SearchService {
|
public class SearchService {
|
||||||
|
|
||||||
public static final int SEARCH_SIZE = 200;
|
public static final int SEARCH_SIZE = 200;
|
||||||
private static BinTrie<List<String>> trie;
|
private static volatile BinTrie<List<String>> trie;
|
||||||
private static BinTrie<List<String>> suffixTrie;
|
private static volatile BinTrie<List<String>> suffixTrie;
|
||||||
|
|
||||||
static {
|
static {
|
||||||
trie = new BinTrie<>();
|
trie = new BinTrie<>();
|
||||||
|
|||||||
@@ -100,8 +100,6 @@ public class HanlpHelper {
|
|||||||
FileHelper.deleteCacheFile(HanLP.Config.CustomDictionaryPath);
|
FileHelper.deleteCacheFile(HanLP.Config.CustomDictionaryPath);
|
||||||
FileHelper.resetCustomPath(getDynamicCustomDictionary());
|
FileHelper.resetCustomPath(getDynamicCustomDictionary());
|
||||||
}
|
}
|
||||||
// 3.clear trie
|
|
||||||
SearchService.clear();
|
|
||||||
|
|
||||||
boolean reload = getDynamicCustomDictionary().reload();
|
boolean reload = getDynamicCustomDictionary().reload();
|
||||||
if (reload) {
|
if (reload) {
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ public class DbAdaptorFactory {
|
|||||||
dbAdaptorMap.put(EngineType.PRESTO.getName(), new PrestoAdaptor());
|
dbAdaptorMap.put(EngineType.PRESTO.getName(), new PrestoAdaptor());
|
||||||
dbAdaptorMap.put(EngineType.TRINO.getName(), new TrinoAdaptor());
|
dbAdaptorMap.put(EngineType.TRINO.getName(), new TrinoAdaptor());
|
||||||
dbAdaptorMap.put(EngineType.ORACLE.getName(), new OracleAdaptor());
|
dbAdaptorMap.put(EngineType.ORACLE.getName(), new OracleAdaptor());
|
||||||
|
dbAdaptorMap.put(EngineType.TDW.getName(), new DefaultDbAdaptor());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DbAdaptor getEngineAdaptor(String engineType) {
|
public static DbAdaptor getEngineAdaptor(String engineType) {
|
||||||
|
|||||||
@@ -43,14 +43,7 @@ public class DataModelNode extends SemanticNode {
|
|||||||
}
|
}
|
||||||
} else if (dataModel.getModelDetail().getTableQuery() != null
|
} else if (dataModel.getModelDetail().getTableQuery() != null
|
||||||
&& !dataModel.getModelDetail().getTableQuery().isEmpty()) {
|
&& !dataModel.getModelDetail().getTableQuery().isEmpty()) {
|
||||||
if (dataModel.getModelDetail().getDbType()
|
sqlTable = "SELECT * FROM " + dataModel.getModelDetail().getTableQuery();
|
||||||
.equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
|
|
||||||
String fullTableName = String.join(".public.",
|
|
||||||
dataModel.getModelDetail().getTableQuery().split("\\."));
|
|
||||||
sqlTable = "SELECT * FROM " + fullTableName;
|
|
||||||
} else {
|
|
||||||
sqlTable = "SELECT * FROM " + dataModel.getModelDetail().getTableQuery();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// String filterSql = dataModel.getFilterSql();
|
// String filterSql = dataModel.getFilterSql();
|
||||||
|
|||||||
@@ -123,7 +123,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
|||||||
|
|
||||||
// 3 translate query
|
// 3 translate query
|
||||||
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
|
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
|
||||||
semanticTranslator.translate(queryStatement);
|
if (!queryStatement.isTranslated()) {
|
||||||
|
semanticTranslator.translate(queryStatement);
|
||||||
|
}
|
||||||
|
|
||||||
// Check whether the dimensions of the metric drill-down are correct temporarily,
|
// Check whether the dimensions of the metric drill-down are correct temporarily,
|
||||||
// add the abstraction of a validator later.
|
// add the abstraction of a validator later.
|
||||||
|
|||||||
@@ -17,12 +17,11 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
|
|||||||
import com.tencent.supersonic.headless.server.utils.DictUtils;
|
import com.tencent.supersonic.headless.server.utils.DictUtils;
|
||||||
import com.xkzhangsan.time.utils.CollectionUtil;
|
import com.xkzhangsan.time.utils.CollectionUtil;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.codehaus.plexus.util.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ public class MetricRepositoryImpl implements MetricRepository {
|
|||||||
if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) {
|
if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) {
|
||||||
queryWrapper.lambda().eq(MetricDO::getCreatedBy, metricFilter.getCreatedBy());
|
queryWrapper.lambda().eq(MetricDO::getCreatedBy, metricFilter.getCreatedBy());
|
||||||
}
|
}
|
||||||
if (Objects.nonNull(metricFilter.getIsPublish()) && metricFilter.getIsPublish() == 1) {
|
if (Objects.nonNull(metricFilter.getIsPublish())) {
|
||||||
queryWrapper.lambda().eq(MetricDO::getIsPublish, metricFilter.getIsPublish());
|
queryWrapper.lambda().eq(MetricDO::getIsPublish, metricFilter.getIsPublish());
|
||||||
}
|
}
|
||||||
if (StringUtils.isNotBlank(metricFilter.getKey())) {
|
if (StringUtils.isNotBlank(metricFilter.getKey())) {
|
||||||
|
|||||||
@@ -46,8 +46,10 @@ public class ChatModelController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@DeleteMapping("/{id}")
|
@DeleteMapping("/{id}")
|
||||||
public boolean deleteModel(@PathVariable("id") Integer id) {
|
public boolean deleteModel(@PathVariable("id") Integer id,
|
||||||
chatModelService.deleteChatModel(id);
|
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
|
||||||
|
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
|
||||||
|
chatModelService.deleteChatModel(id, user);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -64,8 +64,10 @@ public class DatabaseController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@DeleteMapping("/{id}")
|
@DeleteMapping("/{id}")
|
||||||
public boolean deleteDatabase(@PathVariable("id") Long id) {
|
public boolean deleteDatabase(@PathVariable("id") Long id, HttpServletRequest request,
|
||||||
databaseService.deleteDatabase(id);
|
HttpServletResponse response) {
|
||||||
|
User user = UserHolder.findUser(request, response);
|
||||||
|
databaseService.deleteDatabase(id, user);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ public interface DatabaseService {
|
|||||||
|
|
||||||
List<DatabaseResp> getDatabaseList(User user);
|
List<DatabaseResp> getDatabaseList(User user);
|
||||||
|
|
||||||
void deleteDatabase(Long databaseId);
|
void deleteDatabase(Long databaseId, User user);
|
||||||
|
|
||||||
List<String> getCatalogs(Long id) throws SQLException;
|
List<String> getCatalogs(Long id) throws SQLException;
|
||||||
|
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ public interface DimensionService {
|
|||||||
|
|
||||||
List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user);
|
List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user);
|
||||||
|
|
||||||
void sendDimensionEventBatch(List<Long> modelIds, EventType eventType);
|
void sendDimensionEventBatch(List<Long> modelIds, EventType eventType, User user);
|
||||||
|
|
||||||
DataEvent getAllDataEvents();
|
DataEvent getAllDataEvents();
|
||||||
|
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ public interface MetricService {
|
|||||||
|
|
||||||
MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user);
|
MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user);
|
||||||
|
|
||||||
void sendMetricEventBatch(List<Long> modelIds, EventType eventType);
|
void sendMetricEventBatch(List<Long> modelIds, EventType eventType, User user);
|
||||||
|
|
||||||
List<MetricResp> queryMetrics(MetricsFilter metricsFilter);
|
List<MetricResp> queryMetrics(MetricsFilter metricsFilter);
|
||||||
|
|
||||||
|
|||||||
@@ -138,7 +138,12 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteDatabase(Long databaseId) {
|
public void deleteDatabase(Long databaseId, User user) {
|
||||||
|
DatabaseResp databaseResp = getDatabase(databaseId);
|
||||||
|
if (!checkAdminPermission(user, databaseResp)) {
|
||||||
|
throw new RuntimeException("没有权限删除该数据库");
|
||||||
|
}
|
||||||
|
|
||||||
ModelFilter modelFilter = new ModelFilter();
|
ModelFilter modelFilter = new ModelFilter();
|
||||||
modelFilter.setDatabaseId(databaseId);
|
modelFilter.setDatabaseId(databaseId);
|
||||||
modelFilter.setIncludesDetail(false);
|
modelFilter.setIncludesDetail(false);
|
||||||
|
|||||||
@@ -8,7 +8,10 @@ import com.github.pagehelper.PageHelper;
|
|||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.common.pojo.*;
|
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||||
|
import com.tencent.supersonic.common.pojo.DataItem;
|
||||||
|
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||||
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||||
@@ -22,14 +25,22 @@ import com.tencent.supersonic.headless.api.pojo.request.DimValueAliasReq;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
|
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
|
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.*;
|
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||||
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
|
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
|
||||||
import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
|
import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
|
||||||
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
|
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
|
||||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||||
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
||||||
import com.tencent.supersonic.headless.server.service.*;
|
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.DatabaseService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.ModelRelaService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||||
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
||||||
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
||||||
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
|
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
|
||||||
@@ -37,12 +48,17 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.context.ApplicationEventPublisher;
|
import org.springframework.context.ApplicationEventPublisher;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@@ -50,31 +66,31 @@ import java.util.stream.Collectors;
|
|||||||
public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, DimensionDO>
|
public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, DimensionDO>
|
||||||
implements DimensionService {
|
implements DimensionService {
|
||||||
|
|
||||||
private DimensionRepository dimensionRepository;
|
private final DimensionRepository dimensionRepository;
|
||||||
|
|
||||||
private ModelService modelService;
|
private final ModelService modelService;
|
||||||
|
|
||||||
private AliasGenerateHelper aliasGenerateHelper;
|
private final AliasGenerateHelper aliasGenerateHelper;
|
||||||
|
|
||||||
private DatabaseService databaseService;
|
private final DatabaseService databaseService;
|
||||||
|
|
||||||
private ModelRelaService modelRelaService;
|
private final ModelRelaService modelRelaService;
|
||||||
|
|
||||||
private DataSetService dataSetService;
|
private final DataSetService dataSetService;
|
||||||
|
|
||||||
|
private final ApplicationEventPublisher eventPublisher;
|
||||||
@Autowired
|
|
||||||
private ApplicationEventPublisher eventPublisher;
|
|
||||||
|
|
||||||
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
|
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
|
||||||
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
|
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
|
||||||
ModelRelaService modelRelaService, DataSetService dataSetService) {
|
ModelRelaService modelRelaService, DataSetService dataSetService,
|
||||||
|
ApplicationEventPublisher eventPublisher) {
|
||||||
this.modelService = modelService;
|
this.modelService = modelService;
|
||||||
this.dimensionRepository = dimensionRepository;
|
this.dimensionRepository = dimensionRepository;
|
||||||
this.aliasGenerateHelper = aliasGenerateHelper;
|
this.aliasGenerateHelper = aliasGenerateHelper;
|
||||||
this.databaseService = databaseService;
|
this.databaseService = databaseService;
|
||||||
this.modelRelaService = modelRelaService;
|
this.modelRelaService = modelRelaService;
|
||||||
this.dataSetService = dataSetService;
|
this.dataSetService = dataSetService;
|
||||||
|
this.eventPublisher = eventPublisher;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -83,7 +99,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
dimensionReq.createdBy(user.getName());
|
dimensionReq.createdBy(user.getName());
|
||||||
DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimensionReq);
|
DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimensionReq);
|
||||||
dimensionRepository.createDimension(dimensionDO);
|
dimensionRepository.createDimension(dimensionDO);
|
||||||
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD);
|
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD, user);
|
||||||
|
|
||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(),
|
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(),
|
||||||
@@ -101,29 +117,11 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
|
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||||
|
|
||||||
List<DimensionReq> dimensionToInsert = Lists.newArrayList();
|
List<DimensionReq> dimensionToInsert = Lists.newArrayList();
|
||||||
List<DimensionReq> dimensionToUpdate = Lists.newArrayList();
|
|
||||||
List<Long> dimensionToDelete = Lists.newArrayList();
|
|
||||||
|
|
||||||
// look for which dimension need to insert, update, delete
|
// look for which dimension need to insert, update, delete
|
||||||
dimensionReqs.stream().forEach(dimension -> {
|
dimensionReqs.forEach(dimension -> {
|
||||||
if (!bizNameMap.containsKey(dimension.getBizName())) {
|
if (!bizNameMap.containsKey(dimension.getBizName())) {
|
||||||
dimensionToInsert.add(dimension);
|
dimensionToInsert.add(dimension);
|
||||||
} else {
|
|
||||||
DimensionResp dimensionRespByBizName = bizNameMap.get(dimension.getBizName());
|
|
||||||
if (null != dimensionRespByBizName && isChange(dimension, dimensionRespByBizName)) {
|
|
||||||
dimension.setId(dimensionRespByBizName.getId());
|
|
||||||
dimension.updatedBy(user.getName());
|
|
||||||
dimensionToUpdate.add(dimension);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// the bizNames from alter dimensions
|
|
||||||
List<String> bizNames =
|
|
||||||
dimensionReqs.stream().map(DimensionReq::getBizName).collect(Collectors.toList());
|
|
||||||
bizNameMap.keySet().forEach(bizNameInDb -> {
|
|
||||||
if (!bizNames.contains(bizNameInDb)) {
|
|
||||||
dimensionToDelete.add(bizNameMap.get(bizNameInDb).getId());
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -132,16 +130,6 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
createDimensionBatch(dimensionToInsert, user);
|
createDimensionBatch(dimensionToInsert, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
// update
|
|
||||||
if (!CollectionUtils.isEmpty(dimensionToUpdate)) {
|
|
||||||
updateDimensionBatch(dimensionToUpdate, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
// delete
|
|
||||||
if (!CollectionUtils.isEmpty(dimensionToDelete)) {
|
|
||||||
deleteDimensionBatch(dimensionToDelete, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -154,7 +142,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs,
|
modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs,
|
||||||
null, user);
|
null, user);
|
||||||
|
|
||||||
sendEventBatch(dimensionDOS, EventType.ADD);
|
sendEventBatch(dimensionDOS, EventType.ADD, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -170,7 +158,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
Lists.newArrayList(dimensionReq), null, user);
|
Lists.newArrayList(dimensionReq), null, user);
|
||||||
|
|
||||||
if (!oldName.equals(dimensionDO.getName())) {
|
if (!oldName.equals(dimensionDO.getName())) {
|
||||||
sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
|
sendEvent(getDataItem(dimensionDO), EventType.UPDATE, user.getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,7 +171,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),
|
modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),
|
||||||
dimensionReqList, null, user);
|
dimensionReqList, null, user);
|
||||||
sendEventBatch(dimensionDOS, EventType.UPDATE);
|
sendEventBatch(dimensionDOS, EventType.UPDATE, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -205,9 +193,9 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
dimensionRepository.batchUpdateStatus(dimensionDOS);
|
dimensionRepository.batchUpdateStatus(dimensionDOS);
|
||||||
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
||||||
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
||||||
sendEventBatch(dimensionDOS, EventType.DELETE);
|
sendEventBatch(dimensionDOS, EventType.DELETE, user);
|
||||||
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
||||||
sendEventBatch(dimensionDOS, EventType.ADD);
|
sendEventBatch(dimensionDOS, EventType.ADD, user);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -240,7 +228,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(dimensionDO.getModelId(),
|
modelService.deleteModelDetailByDimAndMetric(dimensionDO.getModelId(),
|
||||||
Lists.newArrayList(dimensionDO), null);
|
Lists.newArrayList(dimensionDO), null);
|
||||||
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE);
|
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -261,7 +249,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(),
|
modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(),
|
||||||
dimensionDOList, null);
|
dimensionDOList, null);
|
||||||
sendEventBatch(dimensionDOList, EventType.DELETE);
|
sendEventBatch(dimensionDOList, EventType.DELETE, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -462,22 +450,22 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void sendDimensionEventBatch(List<Long> modelIds, EventType eventType) {
|
public void sendDimensionEventBatch(List<Long> modelIds, EventType eventType, User user) {
|
||||||
DimensionFilter dimensionFilter = new DimensionFilter();
|
DimensionFilter dimensionFilter = new DimensionFilter();
|
||||||
dimensionFilter.setModelIds(modelIds);
|
dimensionFilter.setModelIds(modelIds);
|
||||||
List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
|
List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
|
||||||
sendEventBatch(dimensionDOS, eventType);
|
sendEventBatch(dimensionDOS, eventType, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendEventBatch(List<DimensionDO> dimensionDOS, EventType eventType) {
|
private void sendEventBatch(List<DimensionDO> dimensionDOS, EventType eventType, User user) {
|
||||||
DataEvent dataEvent = getDataEvent(dimensionDOS, eventType);
|
DataEvent dataEvent = getDataEvent(dimensionDOS, eventType, user.getName());
|
||||||
eventPublisher.publishEvent(dataEvent);
|
eventPublisher.publishEvent(dataEvent);
|
||||||
}
|
}
|
||||||
|
|
||||||
public DataEvent getAllDataEvents() {
|
public DataEvent getAllDataEvents() {
|
||||||
DimensionFilter dimensionFilter = new DimensionFilter();
|
DimensionFilter dimensionFilter = new DimensionFilter();
|
||||||
List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
|
List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
|
||||||
return getDataEvent(dimensionDOS, EventType.ADD);
|
return getDataEvent(dimensionDOS, EventType.ADD, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -487,31 +475,53 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
if (StringUtils.isNotEmpty(dimensionDO.getDimValueMaps())) {
|
if (StringUtils.isNotEmpty(dimensionDO.getDimValueMaps())) {
|
||||||
dimValueMapList = JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class);
|
dimValueMapList = JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class);
|
||||||
}
|
}
|
||||||
DimValueMap dimValueMaps = req.getDimValueMaps();
|
|
||||||
if (StringUtils.isEmpty(dimValueMaps.getTechName())) {
|
// 预先处理请求列表,设置默认的 techName
|
||||||
dimValueMaps.setTechName(dimValueMaps.getValue());
|
for (DimValueMap dimValueMap : req.getDimValueMaps()) {
|
||||||
}
|
if (StringUtils.isEmpty(dimValueMap.getTechName())) {
|
||||||
Map<String, DimValueMap> valeAndMapInfo = dimValueMapList.stream()
|
dimValueMap.setTechName(dimValueMap.getValue());
|
||||||
.collect(Collectors.toMap(DimValueMap::getValue, v -> v, (v1, v2) -> v2));
|
|
||||||
String value = dimValueMaps.getValue();
|
|
||||||
if (CollectionUtils.isEmpty(dimValueMaps.getAlias())) {
|
|
||||||
// 删除
|
|
||||||
dimValueMapList =
|
|
||||||
dimValueMapList.stream().filter(map -> !map.getValue().equalsIgnoreCase(value))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
} else {
|
|
||||||
// 新增
|
|
||||||
if (!valeAndMapInfo.keySet().contains(value)) {
|
|
||||||
dimValueMapList.add(dimValueMaps);
|
|
||||||
} else {
|
|
||||||
// 更新
|
|
||||||
dimValueMapList.stream().forEach(map -> {
|
|
||||||
if (map.getValue().equalsIgnoreCase(value)) {
|
|
||||||
map.setAlias(dimValueMaps.getAlias());
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 构建现有数据的 Map,用于快速查找 (value -> DimValueMap)
|
||||||
|
Map<String, DimValueMap> existingMap = dimValueMapList.stream()
|
||||||
|
.collect(Collectors.toMap(DimValueMap::getValue, v -> v, (v1, v2) -> v2));
|
||||||
|
|
||||||
|
// 收集需要删除的 values(alias 为空的)
|
||||||
|
Set<String> valuesToDelete = req.getDimValueMaps().stream()
|
||||||
|
.filter(dimValueMap -> CollectionUtils.isEmpty(dimValueMap.getAlias()))
|
||||||
|
.map(DimValueMap::getValue).collect(Collectors.toSet());
|
||||||
|
|
||||||
|
// 一次性删除所有需要删除的数据
|
||||||
|
if (!valuesToDelete.isEmpty()) {
|
||||||
|
dimValueMapList =
|
||||||
|
dimValueMapList.stream().filter(map -> !valuesToDelete.contains(map.getValue()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
// 同时从 existingMap 中移除
|
||||||
|
existingMap.keySet().removeAll(valuesToDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理新增和更新
|
||||||
|
for (DimValueMap dimValueMap : req.getDimValueMaps()) {
|
||||||
|
// 跳过需要删除的
|
||||||
|
if (CollectionUtils.isEmpty(dimValueMap.getAlias())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
String value = dimValueMap.getValue();
|
||||||
|
if (!existingMap.containsKey(value)) {
|
||||||
|
// 新增
|
||||||
|
dimValueMapList.add(dimValueMap);
|
||||||
|
existingMap.put(value, dimValueMap);
|
||||||
|
} else {
|
||||||
|
// 更新 - 直接更新已存在的对象
|
||||||
|
DimValueMap existing = existingMap.get(value);
|
||||||
|
existing.setAlias(dimValueMap.getAlias());
|
||||||
|
existing.setTechName(dimValueMap.getTechName());
|
||||||
|
existing.setBizName(dimValueMap.getBizName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
dimensionDO.setDimValueMaps(JsonUtil.toString(dimValueMapList));
|
dimensionDO.setDimValueMaps(JsonUtil.toString(dimValueMapList));
|
||||||
updateById(dimensionDO);
|
updateById(dimensionDO);
|
||||||
return true;
|
return true;
|
||||||
@@ -529,14 +539,16 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
.domainId(dimensionResp.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
|
.domainId(dimensionResp.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
|
private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType,
|
||||||
|
String userName) {
|
||||||
List<DataItem> dataItems = dimensionDOS.stream().map(this::getDataItem)
|
List<DataItem> dataItems = dimensionDOS.stream().map(this::getDataItem)
|
||||||
.filter(Objects::nonNull).collect(Collectors.toList());
|
.filter(Objects::nonNull).collect(Collectors.toList());
|
||||||
return new DataEvent(this, dataItems, eventType);
|
return new DataEvent(this, dataItems, eventType, userName);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendEvent(DataItem dataItem, EventType eventType) {
|
private void sendEvent(DataItem dataItem, EventType eventType, String userName) {
|
||||||
eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
|
eventPublisher.publishEvent(
|
||||||
|
new DataEvent(this, Lists.newArrayList(dataItem), eventType, userName));
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isChange(DimensionReq dimensionReq, DimensionResp dimensionResp) {
|
private boolean isChange(DimensionReq dimensionReq, DimensionResp dimensionResp) {
|
||||||
|
|||||||
@@ -8,23 +8,61 @@ import com.github.pagehelper.PageInfo;
|
|||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
import com.tencent.supersonic.common.pojo.*;
|
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||||
import com.tencent.supersonic.common.pojo.enums.*;
|
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||||
|
import com.tencent.supersonic.common.pojo.DataItem;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.Filter;
|
||||||
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||||
import com.tencent.supersonic.common.util.BeanMapper;
|
import com.tencent.supersonic.common.util.BeanMapper;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||||
import com.tencent.supersonic.headless.api.pojo.*;
|
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||||
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
|
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.MetricParam;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||||
import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
|
import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
|
||||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.*;
|
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.*;
|
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||||
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
|
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
|
||||||
import com.tencent.supersonic.headless.server.persistence.dataobject.*;
|
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
|
||||||
|
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||||
|
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
|
||||||
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
|
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
|
||||||
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
||||||
import com.tencent.supersonic.headless.server.pojo.*;
|
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||||
import com.tencent.supersonic.headless.server.service.*;
|
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||||
import com.tencent.supersonic.headless.server.utils.*;
|
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||||
|
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
|
||||||
|
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
||||||
|
import com.tencent.supersonic.headless.server.service.CollectService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||||
|
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
||||||
|
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
|
||||||
|
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||||
|
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
@@ -33,7 +71,18 @@ import org.springframework.context.annotation.Lazy;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@@ -41,21 +90,21 @@ import java.util.stream.Collectors;
|
|||||||
public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||||
implements MetricService {
|
implements MetricService {
|
||||||
|
|
||||||
private MetricRepository metricRepository;
|
private final MetricRepository metricRepository;
|
||||||
|
|
||||||
private ModelService modelService;
|
private final ModelService modelService;
|
||||||
|
|
||||||
private DimensionService dimensionService;
|
private final DimensionService dimensionService;
|
||||||
|
|
||||||
private AliasGenerateHelper aliasGenerateHelper;
|
private final AliasGenerateHelper aliasGenerateHelper;
|
||||||
|
|
||||||
private CollectService collectService;
|
private final CollectService collectService;
|
||||||
|
|
||||||
private DataSetService dataSetService;
|
private final DataSetService dataSetService;
|
||||||
|
|
||||||
private ApplicationEventPublisher eventPublisher;
|
private final ApplicationEventPublisher eventPublisher;
|
||||||
|
|
||||||
private ChatLayerService chatLayerService;
|
private final ChatLayerService chatLayerService;
|
||||||
|
|
||||||
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
|
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
|
||||||
AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
|
AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
|
||||||
@@ -78,7 +127,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
metricReq.createdBy(user.getName());
|
metricReq.createdBy(user.getName());
|
||||||
MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq);
|
MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq);
|
||||||
metricRepository.createMetric(metricDO);
|
metricRepository.createMetric(metricDO);
|
||||||
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
|
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD, user);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
|
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
|
||||||
Lists.newArrayList(metricReq), user);
|
Lists.newArrayList(metricReq), user);
|
||||||
@@ -97,7 +146,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
|
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
|
||||||
user);
|
user);
|
||||||
|
|
||||||
sendEventBatch(metricDOS, EventType.ADD);
|
sendEventBatch(metricDOS, EventType.ADD, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -108,28 +157,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||||
|
|
||||||
List<MetricReq> metricToInsert = Lists.newArrayList();
|
List<MetricReq> metricToInsert = Lists.newArrayList();
|
||||||
List<MetricReq> metricToUpdate = Lists.newArrayList();
|
metricReqs.forEach(metric -> {
|
||||||
List<Long> metricToDelete = Lists.newArrayList();
|
|
||||||
|
|
||||||
metricReqs.stream().forEach(metric -> {
|
|
||||||
if (!bizNameMap.containsKey(metric.getBizName())) {
|
if (!bizNameMap.containsKey(metric.getBizName())) {
|
||||||
metricToInsert.add(metric);
|
metricToInsert.add(metric);
|
||||||
} else {
|
|
||||||
MetricResp metricRespByBizName = bizNameMap.get(metric.getBizName());
|
|
||||||
if (null != metricRespByBizName && isChange(metric, metricRespByBizName)) {
|
|
||||||
metric.setId(metricRespByBizName.getId());
|
|
||||||
metric.updatedBy(user.getName());
|
|
||||||
metricToUpdate.add(metric);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// the bizNames from alter dimensions
|
|
||||||
List<String> bizNames =
|
|
||||||
metricReqs.stream().map(MetricReq::getBizName).collect(Collectors.toList());
|
|
||||||
bizNameMap.keySet().forEach(bizNameInDb -> {
|
|
||||||
if (!bizNames.contains(bizNameInDb)) {
|
|
||||||
metricToDelete.add(bizNameMap.get(bizNameInDb).getId());
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -138,16 +168,6 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
createMetricBatch(metricToInsert, user);
|
createMetricBatch(metricToInsert, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
// update
|
|
||||||
if (!CollectionUtils.isEmpty(metricToUpdate)) {
|
|
||||||
updateMetricBatch(metricToUpdate, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
// delete
|
|
||||||
if (!CollectionUtils.isEmpty(metricToDelete)) {
|
|
||||||
deleteMetricBatch(metricToDelete, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -163,7 +183,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
DataItem dataItem = getDataItem(metricDO);
|
DataItem dataItem = getDataItem(metricDO);
|
||||||
dataItem.setName(oldName);
|
dataItem.setName(oldName);
|
||||||
dataItem.setNewName(metricDO.getName());
|
dataItem.setNewName(metricDO.getName());
|
||||||
sendEvent(dataItem, EventType.UPDATE);
|
sendEvent(dataItem, EventType.UPDATE, user);
|
||||||
}
|
}
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
|
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
|
||||||
@@ -181,7 +201,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
|
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
|
||||||
user);
|
user);
|
||||||
sendEventBatch(metricDOS, EventType.UPDATE);
|
sendEventBatch(metricDOS, EventType.UPDATE, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -202,9 +222,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
metricRepository.batchUpdateStatus(metricDOS);
|
metricRepository.batchUpdateStatus(metricDOS);
|
||||||
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
||||||
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
||||||
sendEventBatch(metricDOS, EventType.DELETE);
|
sendEventBatch(metricDOS, EventType.DELETE, user);
|
||||||
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
||||||
sendEventBatch(metricDOS, EventType.ADD);
|
sendEventBatch(metricDOS, EventType.ADD, user);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -285,7 +305,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null,
|
modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null,
|
||||||
Lists.newArrayList(metricDO));
|
Lists.newArrayList(metricDO));
|
||||||
sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE);
|
sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -306,7 +326,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null,
|
modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null,
|
||||||
metricDOList);
|
metricDOList);
|
||||||
sendEventBatch(metricDOList, EventType.DELETE);
|
sendEventBatch(metricDOList, EventType.DELETE, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -354,7 +374,15 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
MetricFilter metricFilter = new MetricFilter();
|
MetricFilter metricFilter = new MetricFilter();
|
||||||
metricFilter.setUserName(user.getName());
|
metricFilter.setUserName(user.getName());
|
||||||
BeanUtils.copyProperties(pageMetricReq, metricFilter);
|
BeanUtils.copyProperties(pageMetricReq, metricFilter);
|
||||||
if (!CollectionUtils.isEmpty(pageMetricReq.getDomainIds())) {
|
|
||||||
|
// If dataSetId is provided, get models directly from the dataset
|
||||||
|
if (pageMetricReq.getDataSetId() != null) {
|
||||||
|
DataSetResp dataSetResp = dataSetService.getDataSet(pageMetricReq.getDataSetId());
|
||||||
|
if (dataSetResp != null) {
|
||||||
|
pageMetricReq.getModelIds().addAll(dataSetResp.getAllModels());
|
||||||
|
}
|
||||||
|
} else if (!CollectionUtils.isEmpty(pageMetricReq.getDomainIds())) {
|
||||||
|
// Only check domainIds when dataSetId is not provided
|
||||||
List<ModelResp> modelResps =
|
List<ModelResp> modelResps =
|
||||||
modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds());
|
modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds());
|
||||||
List<Long> modelIds =
|
List<Long> modelIds =
|
||||||
@@ -661,11 +689,11 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void sendMetricEventBatch(List<Long> modelIds, EventType eventType) {
|
public void sendMetricEventBatch(List<Long> modelIds, EventType eventType, User user) {
|
||||||
MetricFilter metricFilter = new MetricFilter();
|
MetricFilter metricFilter = new MetricFilter();
|
||||||
metricFilter.setModelIds(modelIds);
|
metricFilter.setModelIds(modelIds);
|
||||||
List<MetricDO> metricDOS = queryMetric(metricFilter);
|
List<MetricDO> metricDOS = queryMetric(metricFilter);
|
||||||
sendEventBatch(metricDOS, eventType);
|
sendEventBatch(metricDOS, eventType, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -678,22 +706,23 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
public DataEvent getDataEvent() {
|
public DataEvent getDataEvent() {
|
||||||
MetricsFilter metricsFilter = new MetricsFilter();
|
MetricsFilter metricsFilter = new MetricsFilter();
|
||||||
List<MetricDO> metricDOS = metricRepository.getMetrics(metricsFilter);
|
List<MetricDO> metricDOS = metricRepository.getMetrics(metricsFilter);
|
||||||
return getDataEvent(metricDOS, EventType.ADD);
|
return getDataEvent(metricDOS, EventType.ADD, User.getDefaultUser());
|
||||||
}
|
}
|
||||||
|
|
||||||
private DataEvent getDataEvent(List<MetricDO> metricDOS, EventType eventType) {
|
private DataEvent getDataEvent(List<MetricDO> metricDOS, EventType eventType, User user) {
|
||||||
List<DataItem> dataItems = metricDOS.stream().map(this::getDataItem)
|
List<DataItem> dataItems = metricDOS.stream().map(this::getDataItem)
|
||||||
.filter(Objects::nonNull).collect(Collectors.toList());
|
.filter(Objects::nonNull).collect(Collectors.toList());
|
||||||
return new DataEvent(this, dataItems, eventType);
|
return new DataEvent(this, dataItems, eventType, user.getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendEventBatch(List<MetricDO> metricDOS, EventType eventType) {
|
private void sendEventBatch(List<MetricDO> metricDOS, EventType eventType, User user) {
|
||||||
DataEvent dataEvent = getDataEvent(metricDOS, eventType);
|
DataEvent dataEvent = getDataEvent(metricDOS, eventType, user);
|
||||||
eventPublisher.publishEvent(dataEvent);
|
eventPublisher.publishEvent(dataEvent);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendEvent(DataItem dataItem, EventType eventType) {
|
private void sendEvent(DataItem dataItem, EventType eventType, User user) {
|
||||||
eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
|
eventPublisher.publishEvent(
|
||||||
|
new DataEvent(this, Lists.newArrayList(dataItem), eventType, user.getName()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private DataItem getDataItem(MetricDO metricDO) {
|
private DataItem getDataItem(MetricDO metricDO) {
|
||||||
|
|||||||
@@ -47,22 +47,9 @@ public class ModelRelaServiceImpl extends ServiceImpl<ModelRelaDOMapper, ModelRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void check(ModelRela modelRela) {
|
private void check(ModelRela modelRela) {
|
||||||
ModelResp fromModel = modelService.getModel(modelRela.getFromModelId());
|
|
||||||
ModelResp toModel = modelService.getModel(modelRela.getToModelId());
|
|
||||||
if (CollectionUtils.isEmpty(modelRela.getJoinConditions())) {
|
if (CollectionUtils.isEmpty(modelRela.getJoinConditions())) {
|
||||||
throw new RuntimeException("关联关系不可为空");
|
throw new RuntimeException("关联关系不可为空");
|
||||||
}
|
}
|
||||||
for (JoinCondition joinCondition : modelRela.getJoinConditions()) {
|
|
||||||
IdentifyType identifyTypeLeft = fromModel.getIdentifyType(joinCondition.getLeftField());
|
|
||||||
IdentifyType identifyTypeRight = toModel.getIdentifyType(joinCondition.getRightField());
|
|
||||||
if (IdentifyType.foreign.equals(identifyTypeLeft)
|
|
||||||
|| IdentifyType.foreign.equals(identifyTypeRight)) {
|
|
||||||
if (!IdentifyType.primary.equals(identifyTypeLeft)
|
|
||||||
&& !IdentifyType.primary.equals(identifyTypeRight)) {
|
|
||||||
throw new RuntimeException("外键必须跟主键关联");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ import java.util.*;
|
|||||||
import java.util.concurrent.*;
|
import java.util.concurrent.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.common.pojo.DimensionConstants.DIMENSION_TIME_FORMAT;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class ModelServiceImpl implements ModelService {
|
public class ModelServiceImpl implements ModelService {
|
||||||
@@ -95,7 +97,7 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
// create or update metric
|
// create or update metric
|
||||||
List<MetricReq> metricReqs = ModelConverter.convertMetricList(modelDO);
|
List<MetricReq> metricReqs = ModelConverter.convertMetricList(modelDO);
|
||||||
metricService.alterMetricBatch(metricReqs, modelDO.getId(), user);
|
metricService.alterMetricBatch(metricReqs, modelDO.getId(), user);
|
||||||
sendEvent(modelDO, EventType.ADD);
|
sendEvent(modelDO, EventType.ADD, user);
|
||||||
return ModelConverter.convert(modelDO);
|
return ModelConverter.convert(modelDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,7 +131,7 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
// create or update metric
|
// create or update metric
|
||||||
List<MetricReq> metricReqs = ModelConverter.convertMetricList(modelDO);
|
List<MetricReq> metricReqs = ModelConverter.convertMetricList(modelDO);
|
||||||
metricService.alterMetricBatch(metricReqs, modelDO.getId(), user);
|
metricService.alterMetricBatch(metricReqs, modelDO.getId(), user);
|
||||||
sendEvent(modelDO, EventType.UPDATE);
|
sendEvent(modelDO, EventType.UPDATE, user);
|
||||||
return ModelConverter.convert(modelDO);
|
return ModelConverter.convert(modelDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -507,14 +509,14 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
||||||
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
||||||
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
|
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||||
EventType.DELETE);
|
EventType.DELETE, user);
|
||||||
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
|
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||||
EventType.DELETE);
|
EventType.DELETE, user);
|
||||||
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
||||||
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
|
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||||
EventType.ADD);
|
EventType.ADD, user);
|
||||||
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
|
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||||
EventType.ADD);
|
EventType.ADD, user);
|
||||||
}
|
}
|
||||||
}).collect(Collectors.toList());
|
}).collect(Collectors.toList());
|
||||||
modelRepository.batchUpdate(modelDOS);
|
modelRepository.batchUpdate(modelDOS);
|
||||||
@@ -530,17 +532,23 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream().filter(
|
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream().filter(
|
||||||
dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
|
dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
|
||||||
.findFirst();
|
.findFirst();
|
||||||
|
String dateFormat = null;
|
||||||
|
if (dimensionReq.getExt().containsKey(DIMENSION_TIME_FORMAT)) {
|
||||||
|
dateFormat = (String) dimensionReq.getExt().get(DIMENSION_TIME_FORMAT);
|
||||||
|
}
|
||||||
if (dimOptional.isPresent()) {
|
if (dimOptional.isPresent()) {
|
||||||
Dimension dimension = dimOptional.get();
|
Dimension dimension = dimOptional.get();
|
||||||
dimension.setExpr(dimensionReq.getExpr());
|
dimension.setExpr(dimensionReq.getExpr());
|
||||||
dimension.setName(dimensionReq.getName());
|
dimension.setName(dimensionReq.getName());
|
||||||
dimension.setType(DimensionType.valueOf(dimensionReq.getType()));
|
dimension.setType(DimensionType.valueOf(dimensionReq.getType()));
|
||||||
dimension.setDescription(dimensionReq.getDescription());
|
dimension.setDescription(dimensionReq.getDescription());
|
||||||
|
dimension.setDateFormat(dateFormat);
|
||||||
} else {
|
} else {
|
||||||
Dimension dimension = Dimension.builder().name(dimensionReq.getName())
|
Dimension dimension = Dimension.builder().name(dimensionReq.getName())
|
||||||
.bizName(dimensionReq.getBizName()).expr(dimensionReq.getExpr())
|
.bizName(dimensionReq.getBizName()).expr(dimensionReq.getExpr())
|
||||||
.type(DimensionType.valueOf(dimensionReq.getType()))
|
.type(DimensionType.valueOf(dimensionReq.getType()))
|
||||||
.description(dimensionReq.getDescription()).build();
|
.dateFormat(dateFormat).description(dimensionReq.getDescription())
|
||||||
|
.build();
|
||||||
modelDetail.getDimensions().add(dimension);
|
modelDetail.getDimensions().add(dimension);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -670,9 +678,10 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendEvent(ModelDO modelDO, EventType eventType) {
|
private void sendEvent(ModelDO modelDO, EventType eventType, User user) {
|
||||||
DataItem dataItem = getDataItem(modelDO);
|
DataItem dataItem = getDataItem(modelDO);
|
||||||
eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
|
eventPublisher.publishEvent(
|
||||||
|
new DataEvent(this, Lists.newArrayList(dataItem), eventType, user.getName()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private DataItem getDataItem(ModelDO modelDO) {
|
private DataItem getDataItem(ModelDO modelDO) {
|
||||||
|
|||||||
@@ -413,11 +413,11 @@ public class SchemaServiceImpl implements SchemaService {
|
|||||||
public List<ItemResp> getDomainDataSetTree() {
|
public List<ItemResp> getDomainDataSetTree() {
|
||||||
List<DomainResp> domainResps = domainService.getDomainList();
|
List<DomainResp> domainResps = domainService.getDomainList();
|
||||||
List<ItemResp> itemResps = domainResps.stream().map(domain -> new ItemResp(domain.getId(),
|
List<ItemResp> itemResps = domainResps.stream().map(domain -> new ItemResp(domain.getId(),
|
||||||
domain.getParentId(), domain.getName(), TypeEnums.DOMAIN))
|
domain.getParentId(), domain.getName(), TypeEnums.DOMAIN)).toList();
|
||||||
.collect(Collectors.toList());
|
|
||||||
Map<Long, ItemResp> itemRespMap =
|
Map<Long, ItemResp> itemRespMap =
|
||||||
itemResps.stream().collect(Collectors.toMap(ItemResp::getId, item -> item));
|
itemResps.stream().collect(Collectors.toMap(ItemResp::getId, item -> item));
|
||||||
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(new MetaFilter());
|
List<DataSetResp> dataSetResps = dataSetService
|
||||||
|
.getDataSetList(MetaFilter.builder().status(StatusEnum.ONLINE.getCode()).build());
|
||||||
for (DataSetResp dataSetResp : dataSetResps) {
|
for (DataSetResp dataSetResp : dataSetResps) {
|
||||||
ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId());
|
ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId());
|
||||||
if (itemResp != null) {
|
if (itemResp != null) {
|
||||||
|
|||||||
@@ -296,6 +296,8 @@ public class DictUtils {
|
|||||||
modelIds.add(dictItemResp.getModelId());
|
modelIds.add(dictItemResp.getModelId());
|
||||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||||
querySqlReq.setSql(sql);
|
querySqlReq.setSql(sql);
|
||||||
|
// bypass semantic translation
|
||||||
|
querySqlReq.getSqlInfo().setQuerySQL(sql);
|
||||||
querySqlReq.setNeedAuth(false);
|
querySqlReq.setNeedAuth(false);
|
||||||
querySqlReq.setModelIds(modelIds);
|
querySqlReq.setModelIds(modelIds);
|
||||||
|
|
||||||
@@ -445,14 +447,29 @@ public class DictUtils {
|
|||||||
if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {
|
if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
// 静态日期
|
// 静态日期
|
||||||
if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) {
|
if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) {
|
||||||
|
String dateFormat = partitionTimeDimension.getDateFormat();
|
||||||
|
if (StringUtils.isEmpty(dateFormat)) {
|
||||||
|
dateFormat = "yyyy-MM-dd"; // 默认格式
|
||||||
|
}
|
||||||
|
|
||||||
|
// 格式化起止日期
|
||||||
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormat);
|
||||||
|
String startDate =
|
||||||
|
LocalDate.parse(config.getDateConf().getStartDate()).format(formatter);
|
||||||
|
String endDate = LocalDate.parse(config.getDateConf().getEndDate()).format(formatter);
|
||||||
|
|
||||||
return String.format("( %s >= '%s' and %s <= '%s' )",
|
return String.format("( %s >= '%s' and %s <= '%s' )",
|
||||||
config.getDateConf().getDateField(), config.getDateConf().getStartDate(),
|
partitionTimeDimension.getBizName(), startDate,
|
||||||
config.getDateConf().getDateField(), config.getDateConf().getEndDate());
|
partitionTimeDimension.getBizName(), endDate);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 动态日期
|
// 动态日期
|
||||||
if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) {
|
if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) {
|
||||||
|
dictItemResp.getConfig().getDateConf()
|
||||||
|
.setDateField(partitionTimeDimension.getBizName());
|
||||||
return generateDictDateFilterRecent(dictItemResp);
|
return generateDictDateFilterRecent(dictItemResp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,14 @@ import {
|
|||||||
SimilarQuestionType,
|
SimilarQuestionType,
|
||||||
} from '../../common/type';
|
} from '../../common/type';
|
||||||
import { createContext, useEffect, useRef, useState } from 'react';
|
import { createContext, useEffect, useRef, useState } from 'react';
|
||||||
import { chatExecute, chatParse, queryData, deleteQuery, switchEntity } from '../../service';
|
import {
|
||||||
|
chatExecute,
|
||||||
|
chatParse,
|
||||||
|
queryData,
|
||||||
|
deleteQuery,
|
||||||
|
switchEntity,
|
||||||
|
getExecuteSummary,
|
||||||
|
} from '../../service';
|
||||||
import { PARSE_ERROR_TIP, PREFIX_CLS, SEARCH_EXCEPTION_TIP } from '../../common/constants';
|
import { PARSE_ERROR_TIP, PREFIX_CLS, SEARCH_EXCEPTION_TIP } from '../../common/constants';
|
||||||
import { message, Spin } from 'antd';
|
import { message, Spin } from 'antd';
|
||||||
import IconFont from '../IconFont';
|
import IconFont from '../IconFont';
|
||||||
@@ -169,7 +176,7 @@ const ChatItem: React.FC<Props> = ({
|
|||||||
setExecuteLoading(true);
|
setExecuteLoading(true);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const res: any = await chatExecute(msg, conversationId!, parseInfoValue, agentId);
|
const res: any = await chatExecute(msg, conversationId!, parseInfoValue, agentId, true);
|
||||||
const valid = updateData(res);
|
const valid = updateData(res);
|
||||||
onMsgDataLoaded?.(
|
onMsgDataLoaded?.(
|
||||||
{
|
{
|
||||||
@@ -180,6 +187,20 @@ const ChatItem: React.FC<Props> = ({
|
|||||||
valid,
|
valid,
|
||||||
isRefresh
|
isRefresh
|
||||||
);
|
);
|
||||||
|
const queryId = parseInfoValue.queryId; // 伪流式 大模型输出
|
||||||
|
if (queryId != undefined && res.data.queryState != 'INVALID') {
|
||||||
|
const getSummary = async (data: any, queryId: number) => {
|
||||||
|
const res2: any = await getExecuteSummary(queryId);
|
||||||
|
if (res2.data.queryMode == null) {
|
||||||
|
res2.data = { ...data, textSummary: res2.data.textSummary };
|
||||||
|
setData(res2.data);
|
||||||
|
setTimeout(() => getSummary(data, queryId), 500);
|
||||||
|
} else {
|
||||||
|
setData(res2.data);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
setTimeout(() => getSummary(res.data, queryId), 500);
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const tip = SEARCH_EXCEPTION_TIP;
|
const tip = SEARCH_EXCEPTION_TIP;
|
||||||
setExecuteTip(SEARCH_EXCEPTION_TIP);
|
setExecuteTip(SEARCH_EXCEPTION_TIP);
|
||||||
@@ -423,6 +444,10 @@ const ChatItem: React.FC<Props> = ({
|
|||||||
return result;
|
return result;
|
||||||
}, {});
|
}, {});
|
||||||
});
|
});
|
||||||
|
if (exportData.length === 0) {
|
||||||
|
message.error('该条消息暂不支持该操作');
|
||||||
|
return;
|
||||||
|
}
|
||||||
exportCsvFile(exportData);
|
exportCsvFile(exportData);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -79,7 +79,8 @@ export function chatExecute(
|
|||||||
queryText: string,
|
queryText: string,
|
||||||
chatId: number,
|
chatId: number,
|
||||||
parseInfo: ChatContextType,
|
parseInfo: ChatContextType,
|
||||||
agentId?: number
|
agentId?: number,
|
||||||
|
streamingResult?:boolean
|
||||||
) {
|
) {
|
||||||
return axios.post<MsgDataType>(`${prefix}/chat/query/execute`, {
|
return axios.post<MsgDataType>(`${prefix}/chat/query/execute`, {
|
||||||
queryText,
|
queryText,
|
||||||
@@ -87,6 +88,15 @@ export function chatExecute(
|
|||||||
chatId: chatId || DEFAULT_CHAT_ID,
|
chatId: chatId || DEFAULT_CHAT_ID,
|
||||||
queryId: parseInfo.queryId,
|
queryId: parseInfo.queryId,
|
||||||
parseId: parseInfo.id,
|
parseId: parseInfo.id,
|
||||||
|
streamingResult:streamingResult
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getExecuteSummary(
|
||||||
|
queryId: number
|
||||||
|
) {
|
||||||
|
return axios.post<MsgDataType>(`${prefix}/chat/query/getExecuteSummary`, {
|
||||||
|
queryId: queryId,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -355,9 +355,11 @@ const DimensionValueSettingModal: React.FC<CreateFormProps> = ({
|
|||||||
onRecordSave={(record) => {
|
onRecordSave={(record) => {
|
||||||
modifyDimensionValue({
|
modifyDimensionValue({
|
||||||
id: dimensionItem.id,
|
id: dimensionItem.id,
|
||||||
dimValueMaps: {
|
dimValueMaps: [
|
||||||
...record,
|
{
|
||||||
},
|
...record,
|
||||||
|
},
|
||||||
|
],
|
||||||
});
|
});
|
||||||
}}
|
}}
|
||||||
hideCtrlBtn={['deleteBtn']}
|
hideCtrlBtn={['deleteBtn']}
|
||||||
|
|||||||
Reference in New Issue
Block a user