topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2026-04-19 13:04:21 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement) Move out the datasource and merge the datasource with the model, and adapt the chat module (#423)

Browse Source 
Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
jipeli
2023-11-27 11:05:24 +08:00
committed by GitHub
parent 0534053ff9
commit 27bb1b322e
190 changed files with 3900 additions and 10561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java
View File

@@ -17,19 +17,20 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import java.util.HashMap;
import java.util.List;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
import java.util.List;
@Slf4j
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@@ -44,7 +45,7 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2SQLReq.setModelId(queryStructReq.getModelId());
queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet());
queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2SQL(queryS2SQLReq, user);
}

32
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java
View File

@@ -7,7 +7,6 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.Entity;
import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
@@ -20,8 +19,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -38,6 +35,7 @@ public class ModelSchemaBuilder {
.alias(SchemaItem.getAliasList(resp.getAlias()))
.build();
modelSchema.setModel(domain);
modelSchema.setModelRelas(resp.getModelRelas());
Set<SchemaElement> metrics = new HashSet<>();
for (MetricSchemaResp metric : resp.getMetrics()) {
@@ -124,23 +122,19 @@ public class ModelSchemaBuilder {
modelSchema.getDimensionValues().addAll(dimensionValues);
modelSchema.getTags().addAll(tags);
Entity entity = resp.getEntity();
if (Objects.nonNull(entity)) {
SchemaElement entityElement = new SchemaElement();
if (!CollectionUtils.isEmpty(entity.getNames()) && Objects.nonNull(entity.getEntityId())) {
Map<Long, SchemaElement> idAndDimPair = dimensions.stream()
.collect(
Collectors.toMap(SchemaElement::getId, schemaElement -> schemaElement, (k1, k2) -> k2));
if (idAndDimPair.containsKey(entity.getEntityId())) {
BeanUtils.copyProperties(idAndDimPair.get(entity.getEntityId()), entityElement);
entityElement.setType(SchemaElementType.ENTITY);
}
entityElement.setAlias(entity.getNames());
modelSchema.setEntity(entityElement);
}
DimSchemaResp dim = resp.getPrimaryKey();
if (dim != null) {
SchemaElement entity = SchemaElement.builder()
.model(resp.getId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
.type(SchemaElementType.ENTITY)
.useCnt(dim.getUseCnt())
.alias(dim.getEntityAlias())
.build();
modelSchema.setEntity(entity);
}
return modelSchema;
}

27
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java
View File

@@ -1,10 +1,5 @@
package com.tencent.supersonic.knowledge.semantic;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
import com.alibaba.fastjson.JSON;
import com.github.pagehelper.PageInfo;
import com.google.gson.Gson;
@@ -31,15 +26,9 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
@@ -53,6 +42,18 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
@Slf4j
public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@@ -73,7 +74,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2SQLReq.setModelId(queryStructReq.getModelId());
queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet());
queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2SQL(queryS2SQLReq, user);
}