topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) support explain in semantic and show explain sql in web and fix chat start error (#103)

Browse Source
This commit is contained in:
lexluo09
2023-09-19 16:38:24 +08:00
committed by GitHub
parent 13dcf0edb9
commit a94a44826b
17 changed files with 367 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java
View File

@@ -8,9 +8,11 @@ import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
@@ -32,14 +34,27 @@ import java.util.List;
public interface SemanticLayer {
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user);
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
List<ModelSchema> getModelSchema();
List<ModelSchema> getModelSchema(List<Long> ids);
ModelSchema getModelSchema(Long model, Boolean cacheEnable);
PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd);
PageInfo<MetricResp> getMetricPage(PageMetricReq pageMetricCmd);
List<DomainResp> getDomainList(User user);
List<ModelResp> getModelList(AuthType authType, Long domainId, User user);
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
}

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.api.component;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import org.apache.calcite.sql.parser.SqlParseException;
/**
@@ -14,6 +15,8 @@ public interface SemanticQuery {
QueryResult execute(User user) throws SqlParseException;
ExplainResp explain(User user);
SemanticParseInfo getParseInfo();
void setParseInfo(SemanticParseInfo parseInfo);

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java
View File

@@ -37,6 +37,8 @@ public class SemanticParseInfo {
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
private Map<String, Object> properties = new HashMap<>();
private EntityInfo entityInfo;
private String sql;
public Long getModelId() {
return model != null ? model.getId() : 0L;
}
@@ -46,6 +48,7 @@ public class SemanticParseInfo {
}
private static class SchemaNameLengthComparator implements Comparator<SchemaElement> {
@Override
public int compare(SchemaElement o1, SchemaElement o2) {
int len1 = o1.getName().length();

35
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java
View File

@@ -15,7 +15,10 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import java.util.ArrayList;
import java.util.List;
@@ -42,12 +45,10 @@ public class DslQuery extends PluginSemanticQuery {
@Override
public QueryResult execute(User user) {
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
LLMResp llmResp = dslParseResult.getLlmResp();
LLMResp llmResp = getLlmResp();
long startTime = System.currentTimeMillis();
QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
QueryDslReq queryDslReq = getQueryDslReq(llmResp);
QueryResultWithSchemaResp queryResp = semanticLayer.queryByDsl(queryDslReq, user);
log.info("queryByDsl cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput());
@@ -71,4 +72,30 @@ public class DslQuery extends PluginSemanticQuery {
parseInfo.setProperties(null);
return queryResult;
}
private LLMResp getLlmResp() {
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
return dslParseResult.getLlmResp();
}
private QueryDslReq getQueryDslReq(LLMResp llmResp) {
QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
return queryDslReq;
}
@Override
public ExplainResp explain(User user) {
ExplainSqlReq explainSqlReq = null;
try {
explainSqlReq = ExplainSqlReq.builder()
.queryTypeEnum(QueryTypeEnum.SQL)
.queryReq(getQueryDslReq(getLlmResp()))
.build();
return semanticLayer.explain(explainSqlReq, user);
} catch (Exception e) {
log.error("explain error explainSqlReq:{}", explainSqlReq, e);
}
return null;
}
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/PluginSemanticQuery.java
View File

@@ -1,7 +1,9 @@
package com.tencent.supersonic.chat.query.plugin;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -17,5 +19,8 @@ public abstract class PluginSemanticQuery implements SemanticQuery {
return parseInfo;
}
@Override
public ExplainResp explain(User user) {
return null;
}
}

19
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -21,8 +21,11 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.io.Serializable;
@@ -215,6 +218,22 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
return queryResult;
}
@Override
public ExplainResp explain(User user) {
ExplainSqlReq explainSqlReq = null;
try {
explainSqlReq = ExplainSqlReq.builder()
.queryTypeEnum(QueryTypeEnum.STRUCT)
.queryReq(convertQueryStruct())
.build();
return semanticLayer.explain(explainSqlReq, user);
} catch (Exception e) {
log.error("explain error explainSqlReq:{}", explainSqlReq, e);
}
return null;
}
public QueryResult multiStructExecute(User user) {
String queryMode = parseInfo.getQueryMode();

40
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -27,6 +27,7 @@ import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.chat.service.StatisticsService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import java.util.List;
import java.util.ArrayList;
import java.util.Set;
@@ -35,9 +36,7 @@ import java.util.Comparator;
import java.util.Objects;
import java.util.stream.Collectors;
//import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
//import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
@@ -64,8 +63,6 @@ public class QueryServiceImpl implements QueryService {
@Autowired
private StatisticsService statisticsService;
private final String entity = "ENTITY";
@Value("${time.threshold: 100}")
private Integer timeThreshold;
@@ -109,12 +106,16 @@ public class QueryServiceImpl implements QueryService {
.map(SemanticQuery::getParseInfo)
.sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
.collect(Collectors.toList());
selectedParses.forEach(parseInfo -> {
if (parseInfo.getQueryMode().contains(entity)) {
String queryMode = parseInfo.getQueryMode();
if (QueryManager.isEntityQuery(queryMode)) {
EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class)
.getEntityInfo(parseInfo, queryReq.getUser());
parseInfo.setEntityInfo(entityInfo);
}
addExplainSql(queryReq, parseInfo);
});
List<SemanticParseInfo> candidateParses = queryCtx.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
@@ -138,6 +139,19 @@ public class QueryServiceImpl implements QueryService {
return parseResult;
}
private void addExplainSql(QueryReq queryReq, SemanticParseInfo parseInfo) {
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (Objects.isNull(semanticQuery)) {
return;
}
semanticQuery.setParseInfo(parseInfo);
ExplainResp explain = semanticQuery.explain(queryReq.getUser());
if (Objects.isNull(explain)) {
return;
}
parseInfo.setSql(explain.getSql());
}
@Override
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
@@ -155,9 +169,9 @@ public class QueryServiceImpl implements QueryService {
chatCtx.setAgentId(queryReq.getAgentId());
Long startTime = System.currentTimeMillis();
QueryResult queryResult = semanticQuery.execute(queryReq.getUser());
Long endTime = System.currentTimeMillis();
if (queryResult != null) {
timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime))
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build());
saveInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(),
queryReq.getUser().getName(), queryReq.getChatId().longValue());
@@ -169,7 +183,6 @@ public class QueryServiceImpl implements QueryService {
}
chatCtx.setQueryText(queryReq.getQueryText());
chatCtx.setUser(queryReq.getUser().getName());
//chatService.addQuery(queryResult, chatCtx);
chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);
} else {
chatService.deleteChatQuery(queryReq.getQueryId());
@@ -179,8 +192,8 @@ public class QueryServiceImpl implements QueryService {
}
public void saveInfo(List<StatisticsDO> timeCostDOList,
String queryText, Long queryId,
String userName, Long chatId) {
String queryText, Long queryId,
String userName, Long chatId) {
List<StatisticsDO> list = timeCostDOList.stream()
.filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList());
list.forEach(o -> {
@@ -264,13 +277,6 @@ public class QueryServiceImpl implements QueryService {
dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
// List<Aggregator> aggregators = new ArrayList<>();
// Aggregator aggregator = new Aggregator(dimensionValueReq.getQueryFilter().getBizName(),
// AggOperatorEnum.DISTINCT);
// aggregators.add(aggregator);
// queryStructReq.setAggregators(aggregators);
queryStructReq.setModelId(dimensionValueReq.getModelId());
queryStructReq.setNativeQuery(true);
List<String> groups = new ArrayList<>();

2
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java
View File

@@ -38,4 +38,6 @@ public class DefaultSemanticConfig {
@Value("${fetchModelList.path:/api/semantic/schema/model/list}")
private String fetchModelListPath;
@Value("${explain.path:/api/semantic/query/explain}")
private String explainPath;
}

8
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java
View File

@@ -8,12 +8,14 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
@@ -95,6 +97,12 @@ public class LocalSemanticLayer extends BaseSemanticLayer {
return schemaService.getModelList(user, authType, domainId);
}
@Override
public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
queryService = ContextUtils.getBean(QueryService.class);
return queryService.explain(explainSqlReq, user);
}
@Override
public PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd) {
dimensionService = ContextUtils.getBean(DimensionService.class);

79
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java
View File

@@ -1,38 +1,44 @@
package com.tencent.supersonic.knowledge.semantic;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
import com.alibaba.fastjson.JSON;
import com.github.pagehelper.PageInfo;
import com.google.gson.Gson;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.ResultData;
import com.tencent.supersonic.common.pojo.ReturnCode;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.exception.CommonException;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.S2ThreadContext;
import com.tencent.supersonic.common.util.ThreadContext;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.exception.CommonException;
import com.tencent.supersonic.common.pojo.ResultData;
import com.tencent.supersonic.common.pojo.ReturnCode;
import java.net.URI;
import java.net.URL;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import java.util.LinkedHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
@@ -45,11 +51,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
@Slf4j
public class RemoteSemanticLayer extends BaseSemanticLayer {
@@ -61,6 +62,10 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
};
private ParameterizedTypeReference<ResultData<ExplainResp>> explainTypeRef =
new ParameterizedTypeReference<ResultData<ExplainResp>>() {
};
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
@@ -130,9 +135,10 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
fillToken(headers);
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
URI requestUrl = UriComponentsBuilder.fromHttpUrl(
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchModelSchemaPath()).build()
.encode().toUri();
String semanticUrl = defaultSemanticConfig.getSemanticUrl();
String fetchModelSchemaPath = defaultSemanticConfig.getFetchModelSchemaPath();
URI requestUrl = UriComponentsBuilder.fromHttpUrl(semanticUrl + fetchModelSchemaPath)
.build().encode().toUri();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(ids);
ParameterizedTypeReference<ResultData<List<ModelSchemaResp>>> responseTypeRef =
@@ -179,6 +185,39 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
return JsonUtil.toList(JsonUtil.toString(domainDescListObject), ModelResp.class);
}
@Override
public <T> ExplainResp explain(ExplainSqlReq<T> explainResp, User user) throws Exception {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
String semanticUrl = defaultSemanticConfig.getSemanticUrl();
String explainPath = defaultSemanticConfig.getExplainPath();
URL url = new URL(new URL(semanticUrl), explainPath);
return explain(url.toString(), JsonUtil.toString(explainResp));
}
public ExplainResp explain(String url, String jsonReq) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
fillToken(headers);
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
HttpEntity<String> entity = new HttpEntity<>(jsonReq, headers);
log.info("url:{},explain:{}", url, entity.getBody());
ResultData<ExplainResp> responseBody;
try {
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
ResponseEntity<ResultData<ExplainResp>> responseEntity = restTemplate.exchange(
requestUrl, HttpMethod.POST, entity, explainTypeRef);
log.info("ApiResponse<ExplainResp> responseBody:{}", responseEntity);
responseBody = responseEntity.getBody();
if (Objects.nonNull(responseBody.getData())) {
return responseBody.getData();
}
return null;
} catch (Exception e) {
throw new RuntimeException("explain interface error,url:" + url, e);
}
}
public Object fetchHttpResult(String url, String bodyJson, HttpMethod httpMethod) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);

7
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java
View File

@@ -75,6 +75,13 @@ class SqlParserSelectHelperTest {
+ "user_id like '%alice%' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, pv FROM s2 WHERE "
+ "user_id like '%alice%' AND publish_date > 10000 "
+ "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
}

135
launchers/chat/src/main/resources/db/chat-schema-h2.sql
View File

@@ -1,3 +1,4 @@
-- chat tables
CREATE TABLE IF NOT EXISTS `s2_chat_context`
(
`chat_id` BIGINT NOT NULL , -- context chat id
@@ -7,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_context`
`semantic_parse` LONGVARCHAR DEFAULT NULL , -- parse data
`ext_data` LONGVARCHAR DEFAULT NULL , -- extend data
PRIMARY KEY (`chat_id`)
);
);
CREATE TABLE IF NOT EXISTS `s2_chat`
(
@@ -21,7 +22,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat`
`is_delete` INT DEFAULT '0' COMMENT 'is deleted',
`is_top` INT DEFAULT '0' COMMENT 'is top',
PRIMARY KEY (`chat_id`)
) ;
) ;
CREATE TABLE `s2_chat_query`
@@ -64,72 +65,21 @@ CREATE TABLE `s2_chat_statistics`
);
CREATE TABLE IF NOT EXISTS `s2_chat_config` (
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT DEFAULT NULL ,
`chat_detail_config` varchar(655) ,
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT DEFAULT NULL ,
`chat_detail_config` varchar(655) ,
`chat_agg_config` varchar(655) ,
`recommended_questions` varchar(1500) ,
`recommended_questions` varchar(1500) ,
`created_at` TIMESTAMP NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_by` varchar(100) NOT NULL ,
`status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted
PRIMARY KEY (`id`)
) ;
-- CREATE TABLE IF NOT EXISTS `s2_chat_config` (
-- `id` INT NOT NULL AUTO_INCREMENT,
-- `domain_id` INT DEFAULT NULL ,
-- `default_metrics` varchar(655) DEFAULT NULL,
-- `visibility` varchar(655) , -- invisible dimension metric information
-- `entity_info` varchar(655) ,
-- `dictionary_info` varchar(655) , -- dictionary-related dimension setting information
-- `created_at` TIMESTAMP NOT NULL ,
-- `updated_at` TIMESTAMP NOT NULL ,
-- `created_by` varchar(100) NOT NULL ,
-- `updated_by` varchar(100) NOT NULL ,
-- `status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted
-- PRIMARY KEY (`id`)
-- ) ;
) ;
COMMENT ON TABLE s2_chat_config IS 'chat config information table ';
CREATE TABLE IF NOT EXISTS `s2_dictionary` (
`id` INT NOT NULL AUTO_INCREMENT,
`domain_id` INT NOT NULL ,
`dim_value_infos` LONGVARCHAR , -- dimension value setting information
`created_at` TIMESTAMP NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted
PRIMARY KEY (`id`),
UNIQUE (domain_id)
);
COMMENT ON TABLE s2_dictionary IS 'dictionary configuration information table';
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`id` INT NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL , -- task name
`description` varchar(255) ,
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`dimension_ids` varchar(500) ,
`status` INT NOT NULL , -- the final status of the task
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
`progress` DOUBLE default 0.00 , -- task real-time progress
`elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
`message` LONGVARCHAR , -- remark related information
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
create table s2_user
(
id INT AUTO_INCREMENT,
@@ -141,6 +91,28 @@ create table s2_user
);
COMMENT ON TABLE s2_user IS 'user information table';
CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` (
`id` INT NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) NOT NULL ,
`model_id` INT NOT NULL ,
`dimensions`LONGVARCHAR ,
`metrics`LONGVARCHAR ,
`orders`LONGVARCHAR ,
`filters`LONGVARCHAR ,
`date_info`LONGVARCHAR ,
`limit` INT NOT NULL ,
`native_query` TINYINT NOT NULL DEFAULT '0' ,
`sql`LONGVARCHAR ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`status` INT NOT NULL ,
`elapsed_ms` bigINT DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_semantic_pasre_info IS 'semantic layer sql parsing information table';
CREATE TABLE IF NOT EXISTS `s2_plugin`
(
`id` INT AUTO_INCREMENT,
@@ -157,5 +129,50 @@ CREATE TABLE IF NOT EXISTS `s2_plugin`
`config` LONGVARCHAR NULL,
`comment` LONGVARCHAR NULL,
PRIMARY KEY (`id`)
); COMMENT ON TABLE s2_plugin IS 'plugin information table';
); COMMENT ON TABLE s2_plugin IS 'plugin information table';
CREATE TABLE IF NOT EXISTS s2_agent
(
id int AUTO_INCREMENT,
name varchar(100) null,
description varchar(500) null,
status int null,
examples varchar(500) null,
config varchar(2000) null,
created_by varchar(100) null,
created_at TIMESTAMP null,
updated_by varchar(100) null,
updated_at TIMESTAMP null,
enable_search int null,
PRIMARY KEY (`id`)
); COMMENT ON TABLE s2_agent IS 'agent information table';
-------demo for semantic and chat
CREATE TABLE IF NOT EXISTS `s2_user_department` (
`user_name` varchar(200) NOT NULL,
`department` varchar(200) NOT NULL -- department of user
);
COMMENT ON TABLE s2_user_department IS 'user_department_info';
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`id` INT NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL , -- task name
`description` varchar(255) ,
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`status` INT NOT NULL , -- the final status of the task
`dimension_ids` varchar(500) NULL ,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
`progress` DOUBLE default 0.00 , -- task real-time progress
`elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
`message` LONGVARCHAR , -- remark related information
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';

19
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ExplainResp.java Normal file
View File

@@ -0,0 +1,19 @@
package com.tencent.supersonic.semantic.api.model.response;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@ToString
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ExplainResp implements Serializable {
private String sql;
}

20
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/ExplainSqlReq.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.semantic.api.query.request;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@ToString
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ExplainSqlReq<T> {
private QueryTypeEnum queryTypeEnum;
private T queryReq;
}

44
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -2,18 +2,22 @@ package com.tencent.supersonic.semantic.query.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -89,10 +93,36 @@ public class QueryController {
@PostMapping("/queryDimValue")
public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
HttpServletRequest request,
HttpServletResponse response) {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return queryService.queryDimValue(queryDimValueReq, user);
}
@PostMapping("/explain")
public <T> ExplainResp explain(@RequestBody ExplainSqlReq<T> explainSqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq());
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryTypeEnum.SQL.equals(queryTypeEnum)) {
QueryDslReq queryDslReq = JsonUtil.toObject(queryReqJson, QueryDslReq.class);
ExplainSqlReq<QueryDslReq> explainSqlReqNew = ExplainSqlReq.<QueryDslReq>builder()
.queryReq(queryDslReq)
.queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user);
}
if (QueryTypeEnum.STRUCT.equals(queryTypeEnum)) {
QueryStructReq queryStructReq = JsonUtil.toObject(queryReqJson, QueryStructReq.class);
ExplainSqlReq<QueryStructReq> explainSqlReqNew = ExplainSqlReq.<QueryStructReq>builder()
.queryReq(queryStructReq)
.queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user);
}
return null;
}
}

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -1,11 +1,13 @@
package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import java.util.List;
@@ -25,4 +27,5 @@ public interface QueryService {
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
}

36
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -6,12 +6,15 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
@@ -64,6 +67,11 @@ public class QueryServiceImpl implements QueryService {
@Override
public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception {
QueryStatement queryStatement = convertToQueryStatement(querySqlCmd, user);
return semanticQueryEngine.execute(queryStatement);
}
private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId());
@@ -74,7 +82,7 @@ public class QueryServiceImpl implements QueryService {
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchemas);
queryStatement.setModelId(querySqlCmd.getModelId());
return semanticQueryEngine.execute(queryStatement);
return queryStatement;
}
@Override
@@ -183,6 +191,32 @@ public class QueryServiceImpl implements QueryService {
return statInfos;
}
@Override
public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq();
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryDslReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryDslReq) queryReq, user);
return getExplainResp(queryStatement);
}
if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {
QueryStatement queryStatement = semanticQueryEngine.plan((QueryStructReq) queryReq);
return getExplainResp(queryStatement);
}
throw new IllegalArgumentException("Parameters are invalid, explainSqlReq: " + explainSqlReq);
}
private ExplainResp getExplainResp(QueryStatement queryStatement) {
String sql = "";
if (Objects.nonNull(queryStatement)) {
sql = queryStatement.getSql();
}
return ExplainResp.builder().sql(sql).build();
}
private boolean isCache(QueryStructReq queryStructCmd) {
if (!cacheEnable) {
return false;