topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 04:27:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](supersonic) based on version 0.7.2 (#34)

Browse Source 
Co-authored-by: zuopengge <hwzuopengge@tencent.com>
This commit is contained in:
mainmain
2023-08-20 17:30:35 +08:00
committed by GitHub
parent c93e60ced7
commit cf1b5336c3
122 changed files with 4045 additions and 1075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DimensionWordBuilder.java
View File

@@ -1,13 +1,17 @@
package com.tencent.supersonic.knowledge.dictionary.builder;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
/**
* dimension word nature
@@ -23,6 +27,7 @@ public class DimensionWordBuilder extends BaseWordBuilder {
public List<DictWord> doGet(String word, SchemaElement schemaElement) {
List<DictWord> result = Lists.newArrayList();
result.add(getOnwWordNature(word, schemaElement, false));
result.addAll(getOnwWordNatureAlias(schemaElement, false));
if (nlpDimensionUseSuffix) {
String reverseWord = StringUtils.reverse(word);
if (StringUtils.isNotEmpty(word) && !word.equalsIgnoreCase(reverseWord)) {
@@ -46,4 +51,16 @@ public class DimensionWordBuilder extends BaseWordBuilder {
return dictWord;
}
private List<DictWord> getOnwWordNatureAlias(SchemaElement schemaElement, boolean isSuffix) {
List<DictWord> dictWords = new ArrayList<>();
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
return dictWords;
}
for (String alias : schemaElement.getAlias()) {
dictWords.add(getOnwWordNature(alias, schemaElement, false));
}
return dictWords;
}
}

19
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/MetricWordBuilder.java
View File

@@ -1,13 +1,17 @@
package com.tencent.supersonic.knowledge.dictionary.builder;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
/**
* Metric DictWord
@@ -22,6 +26,7 @@ public class MetricWordBuilder extends BaseWordBuilder {
public List<DictWord> doGet(String word, SchemaElement schemaElement) {
List<DictWord> result = Lists.newArrayList();
result.add(getOnwWordNature(word, schemaElement, false));
result.addAll(getOnwWordNatureAlias(schemaElement, false));
if (nlpMetricUseSuffix) {
String reverseWord = StringUtils.reverse(word);
if (!word.equalsIgnoreCase(reverseWord)) {
@@ -45,4 +50,16 @@ public class MetricWordBuilder extends BaseWordBuilder {
return dictWord;
}
private List<DictWord> getOnwWordNatureAlias(SchemaElement schemaElement, boolean isSuffix) {
List<DictWord> dictWords = new ArrayList<>();
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
return dictWords;
}
for (String alias : schemaElement.getAlias()) {
dictWords.add(getOnwWordNature(alias, schemaElement, false));
}
return dictWords;
}
}

27
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java
View File

@@ -2,51 +2,38 @@ package com.tencent.supersonic.knowledge.semantic;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.S2ThreadContext;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.*;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import java.util.List;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LocalSemanticLayer extends BaseSemanticLayer {
private SchemaService schemaService;
private S2ThreadContext s2ThreadContext;
private DomainService domainService;
private ModelService modelService;
private DimensionService dimensionService;
private MetricService metricService;
@SneakyThrows
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
try {
QueryService queryService = ContextUtils.getBean(QueryService.class);
QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user);
return queryResultWithSchemaResp;
} catch (Exception e) {
log.info("queryByStruct has an exception:{}", e.toString());
}
return null;
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user){
QueryService queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByStructWithAuth(queryStructReq, user);
}
@Override

45
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java
View File

@@ -8,21 +8,17 @@ import com.tencent.supersonic.semantic.api.model.pojo.Entity;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
public class ModelSchemaBuilder {
private static String aliasSplit = ",";
public static ModelSchema build(ModelSchemaResp resp) {
ModelSchema domainSchema = new ModelSchema();
@@ -37,6 +33,13 @@ public class ModelSchemaBuilder {
Set<SchemaElement> metrics = new HashSet<>();
for (MetricSchemaResp metric : resp.getMetrics()) {
List<String> alias = new ArrayList<>();
String aliasStr = metric.getAlias();
if (Strings.isNotEmpty(aliasStr)) {
alias = Arrays.asList(aliasStr.split(aliasSplit));
}
SchemaElement metricToAdd = SchemaElement.builder()
.model(resp.getId())
.id(metric.getId())
@@ -44,16 +47,10 @@ public class ModelSchemaBuilder {
.bizName(metric.getBizName())
.type(SchemaElementType.METRIC)
.useCnt(metric.getUseCnt())
.alias(alias)
.build();
metrics.add(metricToAdd);
String alias = metric.getAlias();
if (StringUtils.isNotEmpty(alias)) {
SchemaElement alisMetricToAdd = new SchemaElement();
BeanUtils.copyProperties(metricToAdd, alisMetricToAdd);
alisMetricToAdd.setName(alias);
metrics.add(alisMetricToAdd);
}
}
domainSchema.getMetrics().addAll(metrics);
@@ -74,6 +71,11 @@ public class ModelSchemaBuilder {
}
}
List<String> alias = new ArrayList<>();
String aliasStr = dim.getAlias();
if (Strings.isNotEmpty(aliasStr)) {
alias = Arrays.asList(aliasStr.split(aliasSplit));
}
SchemaElement dimToAdd = SchemaElement.builder()
.model(resp.getId())
.id(dim.getId())
@@ -81,17 +83,10 @@ public class ModelSchemaBuilder {
.bizName(dim.getBizName())
.type(SchemaElementType.DIMENSION)
.useCnt(dim.getUseCnt())
.alias(alias)
.build();
dimensions.add(dimToAdd);
String alias = dim.getAlias();
if (StringUtils.isNotEmpty(alias)) {
SchemaElement alisDimToAdd = new SchemaElement();
BeanUtils.copyProperties(dimToAdd, alisDimToAdd);
alisDimToAdd.setName(alias);
dimensions.add(alisDimToAdd);
}
SchemaElement dimValueToAdd = SchemaElement.builder()
.model(resp.getId())
.id(dim.getId())
@@ -115,7 +110,7 @@ public class ModelSchemaBuilder {
.collect(
Collectors.toMap(SchemaElement::getId, schemaElement -> schemaElement, (k1, k2) -> k2));
if (idAndDimPair.containsKey(entity.getEntityId())) {
entityElement = idAndDimPair.get(entity.getEntityId());
BeanUtils.copyProperties(idAndDimPair.get(entity.getEntityId()), entityElement);
entityElement.setType(SchemaElementType.ENTITY);
}
entityElement.setAlias(entity.getNames());