mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 12:07:42 +00:00
[improvement][project] supersonic 0.7.2 version backend update (#28)
Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
@@ -10,7 +10,6 @@ import com.hankcs.hanlp.corpus.io.IOUtil;
|
||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||
import com.hankcs.hanlp.utility.Predefine;
|
||||
import com.hankcs.hanlp.utility.TextUtility;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.DataOutputStream;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge;
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
import com.tencent.supersonic.knowledge.service.KnowledgeService;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
import com.tencent.supersonic.knowledge.service.WordService;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -12,8 +13,6 @@ import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class ApplicationStartedInit implements ApplicationListener<ApplicationStartedEvent> {
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
|
||||
@Data
|
||||
public class DictConfig {
|
||||
|
||||
private Long domainId;
|
||||
private Long modelId;
|
||||
|
||||
private List<DimValueInfo> dimValueInfoList;
|
||||
}
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.knowledge.dictionary;
|
||||
public enum DictUpdateMode {
|
||||
|
||||
OFFLINE_FULL("OFFLINE_FULL"),
|
||||
OFFLINE_DOMAIN("OFFLINE_DOMAIN"),
|
||||
OFFLINE_MODEL("OFFLINE_MODEL"),
|
||||
REALTIME_ADD("REALTIME_ADD"),
|
||||
REALTIME_DELETE("REALTIME_DELETE"),
|
||||
NOT_SUPPORT("NOT_SUPPORT");
|
||||
|
||||
@@ -12,6 +12,7 @@ public enum DictWordType {
|
||||
VALUE("value"),
|
||||
|
||||
DOMAIN("dm"),
|
||||
MODEL("model"),
|
||||
ENTITY("entity"),
|
||||
|
||||
NUMBER("m"),
|
||||
@@ -31,7 +32,6 @@ public enum DictWordType {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static DictWordType getNatureType(String nature) {
|
||||
if (StringUtils.isEmpty(nature) || !nature.startsWith(NATURE_SPILT)) {
|
||||
return null;
|
||||
|
||||
@@ -4,8 +4,6 @@ package com.tencent.supersonic.knowledge.dictionary;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@@ -13,7 +11,7 @@ public class DimValue2DictCommand {
|
||||
|
||||
private DictUpdateMode updateMode;
|
||||
|
||||
private List<Long> domainIds;
|
||||
private List<Long> modelIds;
|
||||
|
||||
private Map<Long, List<Long>> domainAndDimPair = new HashMap<>();
|
||||
private Map<Long, List<Long>> modelAndDimPair = new HashMap<>();
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.URI;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary;
|
||||
|
||||
import com.google.common.base.Objects;
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@@ -36,13 +36,13 @@ public class MapResult implements Serializable {
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
MapResult that = (MapResult) o;
|
||||
return Objects.equals(name, that.name) && Objects.equals(natures, that.natures);
|
||||
MapResult mapResult = (MapResult) o;
|
||||
return Objects.equal(name, mapResult.name) && Objects.equal(natures, mapResult.natures);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, natures);
|
||||
return Objects.hashCode(name, natures);
|
||||
}
|
||||
|
||||
public void setOffset(int offset) {
|
||||
|
||||
@@ -16,7 +16,6 @@ import com.hankcs.hanlp.utility.Predefine;
|
||||
import com.hankcs.hanlp.utility.TextUtility;
|
||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.DataOutputStream;
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -37,7 +35,7 @@ public class DimensionWordBuilder extends BaseWordBuilder {
|
||||
private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) {
|
||||
DictWord dictWord = new DictWord();
|
||||
dictWord.setWord(word);
|
||||
Long domainId = schemaElement.getDomain();
|
||||
Long domainId = schemaElement.getModel();
|
||||
String nature = DictWordType.NATURE_SPILT + domainId + DictWordType.NATURE_SPILT + schemaElement.getId()
|
||||
+ DictWordType.DIMENSION.getType();
|
||||
if (isSuffix) {
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -22,8 +20,8 @@ public class DomainWordBuilder extends BaseWordBuilder {
|
||||
List<DictWord> result = Lists.newArrayList();
|
||||
DictWord dictWord = new DictWord();
|
||||
dictWord.setWord(word);
|
||||
Long domainId = schemaElement.getDomain();
|
||||
String nature = DictWordType.NATURE_SPILT + domainId;
|
||||
Long modelId = schemaElement.getModel();
|
||||
String nature = DictWordType.NATURE_SPILT + modelId;
|
||||
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
|
||||
result.add(dictWord);
|
||||
return result;
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -27,7 +25,7 @@ public class EntityWordBuilder extends BaseWordBuilder {
|
||||
return result;
|
||||
}
|
||||
|
||||
Long domain = schemaElement.getDomain();
|
||||
Long domain = schemaElement.getModel();
|
||||
String nature = DictWordType.NATURE_SPILT + domain + DictWordType.NATURE_SPILT + schemaElement.getId()
|
||||
+ DictWordType.ENTITY.getType();
|
||||
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -36,11 +34,11 @@ public class MetricWordBuilder extends BaseWordBuilder {
|
||||
private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) {
|
||||
DictWord dictWord = new DictWord();
|
||||
dictWord.setWord(word);
|
||||
Long domainId = schemaElement.getDomain();
|
||||
String nature = DictWordType.NATURE_SPILT + domainId + DictWordType.NATURE_SPILT + schemaElement.getId()
|
||||
Long modelId = schemaElement.getModel();
|
||||
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
|
||||
+ DictWordType.METRIC.getType();
|
||||
if (isSuffix) {
|
||||
nature = DictWordType.NATURE_SPILT + domainId + DictWordType.NATURE_SPILT + schemaElement.getId()
|
||||
nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
|
||||
+ DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType();
|
||||
}
|
||||
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -27,8 +25,8 @@ public class ValueWordBuilder extends BaseWordBuilder {
|
||||
|
||||
schemaElement.getAlias().stream().forEach(value -> {
|
||||
DictWord dictWord = new DictWord();
|
||||
Long domainId = schemaElement.getDomain();
|
||||
String nature = DictWordType.NATURE_SPILT + domainId + DictWordType.NATURE_SPILT + schemaElement.getId();
|
||||
Long modelId = schemaElement.getModel();
|
||||
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId();
|
||||
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
|
||||
dictWord.setWord(value);
|
||||
result.add(dictWord);
|
||||
|
||||
@@ -2,8 +2,6 @@ package com.tencent.supersonic.knowledge.dictionary.builder;
|
||||
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.dictionary.builder.*;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ public class DictConfDO {
|
||||
|
||||
private Long id;
|
||||
|
||||
private Long domainId;
|
||||
private Long modelId;
|
||||
|
||||
private String dimValueInfos;
|
||||
|
||||
|
||||
@@ -13,5 +13,5 @@ public interface DictConfMapper {
|
||||
|
||||
Boolean upsertDictInfo(DictConfDO dictConfDO);
|
||||
|
||||
DictConfDO getDictInfoByDomainId(Long domainId);
|
||||
DictConfDO getDictInfoByModelId(Long modelId);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.knowledge.persistence.mapper;
|
||||
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import java.util.List;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge.persistence.repository;
|
||||
|
||||
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import java.util.List;
|
||||
|
||||
public interface DictRepository {
|
||||
@@ -15,5 +15,5 @@ public interface DictRepository {
|
||||
|
||||
List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter);
|
||||
|
||||
DictConfig getDictInfoByDomainId(Long domainId);
|
||||
DictConfig getDictInfoByModelId(Long modelId);
|
||||
}
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
package com.tencent.supersonic.knowledge.persistence.repository;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import com.tencent.supersonic.knowledge.persistence.mapper.DictConfMapper;
|
||||
import com.tencent.supersonic.knowledge.persistence.mapper.DictTaskMapper;
|
||||
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
@@ -67,8 +67,8 @@ public class DictRepositoryImpl implements DictRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public DictConfig getDictInfoByDomainId(Long domainId) {
|
||||
DictConfDO dictConfDO = dictConfMapper.getDictInfoByDomainId(domainId);
|
||||
public DictConfig getDictInfoByModelId(Long modelId) {
|
||||
DictConfDO dictConfDO = dictConfMapper.getDictInfoByModelId(modelId);
|
||||
if (Objects.isNull(dictConfDO)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -3,9 +3,9 @@ package com.tencent.supersonic.knowledge.semantic;
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||
import com.tencent.supersonic.common.pojo.ResultData;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@@ -19,7 +19,7 @@ import org.springframework.util.CollectionUtils;
|
||||
@Slf4j
|
||||
public abstract class BaseSemanticLayer implements SemanticLayer {
|
||||
|
||||
protected final Cache<String, List<DomainSchemaResp>> domainSchemaCache =
|
||||
protected final Cache<String, List<ModelSchemaResp>> modelSchemaCache =
|
||||
CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build();
|
||||
|
||||
protected ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>> structTypeRef =
|
||||
@@ -27,48 +27,48 @@ public abstract class BaseSemanticLayer implements SemanticLayer {
|
||||
};
|
||||
|
||||
@SneakyThrows
|
||||
public List<DomainSchemaResp> fetchDomainSchema(List<Long> ids, Boolean cacheEnable) {
|
||||
public List<ModelSchemaResp> fetchModelSchema(List<Long> ids, Boolean cacheEnable) {
|
||||
if (cacheEnable) {
|
||||
return domainSchemaCache.get(String.valueOf(ids), () -> {
|
||||
List<DomainSchemaResp> data = doFetchDomainSchema(ids);
|
||||
return modelSchemaCache.get(String.valueOf(ids), () -> {
|
||||
List<ModelSchemaResp> data = doFetchModelSchema(ids);
|
||||
return data;
|
||||
});
|
||||
}
|
||||
List<DomainSchemaResp> data = doFetchDomainSchema(ids);
|
||||
List<ModelSchemaResp> data = doFetchModelSchema(ids);
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DomainSchema getDomainSchema(Long domain, Boolean cacheEnable) {
|
||||
public ModelSchema getModelSchema(Long domain, Boolean cacheEnable) {
|
||||
List<Long> ids = new ArrayList<>();
|
||||
ids.add(domain);
|
||||
List<DomainSchemaResp> domainSchemaResps = fetchDomainSchema(ids, cacheEnable);
|
||||
if (!CollectionUtils.isEmpty(domainSchemaResps)) {
|
||||
Optional<DomainSchemaResp> domainSchemaResp = domainSchemaResps.stream()
|
||||
List<ModelSchemaResp> modelSchemaResps = fetchModelSchema(ids, cacheEnable);
|
||||
if (!CollectionUtils.isEmpty(modelSchemaResps)) {
|
||||
Optional<ModelSchemaResp> modelSchemaResp = modelSchemaResps.stream()
|
||||
.filter(d -> d.getId().equals(domain)).findFirst();
|
||||
if (domainSchemaResp.isPresent()) {
|
||||
DomainSchemaResp domainSchema = domainSchemaResp.get();
|
||||
return DomainSchemaBuilder.build(domainSchema);
|
||||
if (modelSchemaResp.isPresent()) {
|
||||
ModelSchemaResp modelSchema = modelSchemaResp.get();
|
||||
return ModelSchemaBuilder.build(modelSchema);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainSchema> getDomainSchema() {
|
||||
return getDomainSchema(new ArrayList<>());
|
||||
public List<ModelSchema> getModelSchema() {
|
||||
return getModelSchema(new ArrayList<>());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainSchema> getDomainSchema(List<Long> ids) {
|
||||
List<DomainSchema> domainSchemaList = new ArrayList<>();
|
||||
public List<ModelSchema> getModelSchema(List<Long> ids) {
|
||||
List<ModelSchema> domainSchemaList = new ArrayList<>();
|
||||
|
||||
for(DomainSchemaResp resp : fetchDomainSchema(ids, true)) {
|
||||
domainSchemaList.add(DomainSchemaBuilder.build(resp));
|
||||
for (ModelSchemaResp resp : fetchModelSchema(ids, true)) {
|
||||
domainSchemaList.add(ModelSchemaBuilder.build(resp));
|
||||
}
|
||||
|
||||
return domainSchemaList;
|
||||
}
|
||||
|
||||
protected abstract List<DomainSchemaResp> doFetchDomainSchema(List<Long> ids);
|
||||
protected abstract List<ModelSchemaResp> doFetchModelSchema(List<Long> ids);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package com.tencent.supersonic.knowledge.semantic;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
@Configuration
|
||||
@Data
|
||||
@@ -22,19 +20,19 @@ public class DefaultSemanticConfig {
|
||||
@Value("${searchByStruct.path:/api/semantic/query/sql}")
|
||||
private String searchBySqlPath;
|
||||
|
||||
@Value("${fetchDomainSchemaPath.path:/api/semantic/schema}")
|
||||
private String fetchDomainSchemaPath;
|
||||
@Value("${fetchModelSchemaPath.path:/api/semantic/schema}")
|
||||
private String fetchModelSchemaPath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/dimension/page}")
|
||||
@Value("${fetchModelList.path:/api/semantic/schema/dimension/page}")
|
||||
private String fetchDimensionPagePath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/metric/page}")
|
||||
@Value("${fetchModelList.path:/api/semantic/schema/metric/page}")
|
||||
private String fetchMetricPagePath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/domain/list}")
|
||||
@Value("${fetchModelList.path:/api/semantic/schema/domain/list}")
|
||||
private String fetchDomainListPath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/domain/view/list}")
|
||||
private String fetchDomainViewListPath;
|
||||
@Value("${fetchModelList.path:/api/semantic/schema/model/list}")
|
||||
private String fetchModelListPath;
|
||||
|
||||
}
|
||||
|
||||
@@ -2,17 +2,18 @@ package com.tencent.supersonic.knowledge.semantic;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||
import com.tencent.supersonic.common.util.ThreadContext;
|
||||
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
||||
@@ -20,6 +21,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.model.domain.DimensionService;
|
||||
import com.tencent.supersonic.semantic.model.domain.DomainService;
|
||||
import com.tencent.supersonic.semantic.model.domain.MetricService;
|
||||
import com.tencent.supersonic.semantic.model.domain.ModelService;
|
||||
import com.tencent.supersonic.semantic.query.service.QueryService;
|
||||
import com.tencent.supersonic.semantic.query.service.SchemaService;
|
||||
import java.util.List;
|
||||
@@ -31,6 +33,7 @@ public class LocalSemanticLayer extends BaseSemanticLayer {
|
||||
private SchemaService schemaService;
|
||||
private S2ThreadContext s2ThreadContext;
|
||||
private DomainService domainService;
|
||||
private ModelService modelService;
|
||||
private DimensionService dimensionService;
|
||||
private MetricService metricService;
|
||||
|
||||
@@ -72,27 +75,23 @@ public class LocalSemanticLayer extends BaseSemanticLayer {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainSchemaResp> doFetchDomainSchema(List<Long> ids) {
|
||||
DomainSchemaFilterReq filter = new DomainSchemaFilterReq();
|
||||
filter.setDomainIds(ids);
|
||||
User user = new User(1L, "admin", "admin", "admin@email");
|
||||
schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
return schemaService.fetchDomainSchema(filter, user);
|
||||
}
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForViewer() {
|
||||
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
|
||||
ThreadContext threadContext = s2ThreadContext.get();
|
||||
domainService = ContextUtils.getBean(DomainService.class);
|
||||
return domainService.getDomainListForViewer(threadContext.getUserName());
|
||||
public List<ModelSchemaResp> doFetchModelSchema(List<Long> ids) {
|
||||
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
|
||||
filter.setModelIds(ids);
|
||||
modelService = ContextUtils.getBean(ModelService.class);
|
||||
return modelService.fetchModelSchema(filter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForAdmin() {
|
||||
domainService = ContextUtils.getBean(DomainService.class);
|
||||
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
|
||||
ThreadContext threadContext = s2ThreadContext.get();
|
||||
return domainService.getDomainListForAdmin(threadContext.getUserName());
|
||||
public List<DomainResp> getDomainList(User user) {
|
||||
schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
return schemaService.getDomainList(user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ModelResp> getModelList(AuthType authType, Long domainId, User user) {
|
||||
schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
return schemaService.getModelList(user, authType, domainId);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -102,9 +101,9 @@ public class LocalSemanticLayer extends BaseSemanticLayer {
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<MetricResp> getMetricPage(PageMetricReq pageMetricCmd) {
|
||||
public PageInfo<MetricResp> getMetricPage(PageMetricReq pageMetricReq) {
|
||||
metricService = ContextUtils.getBean(MetricService.class);
|
||||
return metricService.queryMetric(pageMetricCmd);
|
||||
return metricService.queryMetric(pageMetricReq);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,39 +1,44 @@
|
||||
package com.tencent.supersonic.knowledge.semantic;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.Entity;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
public class ModelSchemaBuilder {
|
||||
|
||||
public class DomainSchemaBuilder {
|
||||
|
||||
public static DomainSchema build(DomainSchemaResp resp) {
|
||||
DomainSchema domainSchema = new DomainSchema();
|
||||
public static ModelSchema build(ModelSchemaResp resp) {
|
||||
ModelSchema domainSchema = new ModelSchema();
|
||||
|
||||
SchemaElement domain = SchemaElement.builder()
|
||||
.domain(resp.getId())
|
||||
.model(resp.getId())
|
||||
.id(resp.getId())
|
||||
.name(resp.getName())
|
||||
.bizName(resp.getBizName())
|
||||
.type(SchemaElementType.DOMAIN)
|
||||
.type(SchemaElementType.MODEL)
|
||||
.build();
|
||||
domainSchema.setDomain(domain);
|
||||
domainSchema.setModel(domain);
|
||||
|
||||
Set<SchemaElement> metrics = new HashSet<>();
|
||||
for (MetricSchemaResp metric : resp.getMetrics()) {
|
||||
SchemaElement metricToAdd = SchemaElement.builder()
|
||||
.domain(resp.getId())
|
||||
.model(resp.getId())
|
||||
.id(metric.getId())
|
||||
.name(metric.getName())
|
||||
.bizName(metric.getBizName())
|
||||
@@ -70,7 +75,7 @@ public class DomainSchemaBuilder {
|
||||
}
|
||||
|
||||
SchemaElement dimToAdd = SchemaElement.builder()
|
||||
.domain(resp.getId())
|
||||
.model(resp.getId())
|
||||
.id(dim.getId())
|
||||
.name(dim.getName())
|
||||
.bizName(dim.getBizName())
|
||||
@@ -87,9 +92,8 @@ public class DomainSchemaBuilder {
|
||||
dimensions.add(alisDimToAdd);
|
||||
}
|
||||
|
||||
|
||||
SchemaElement dimValueToAdd = SchemaElement.builder()
|
||||
.domain(resp.getId())
|
||||
.model(resp.getId())
|
||||
.id(dim.getId())
|
||||
.name(dim.getName())
|
||||
.bizName(dim.getBizName())
|
||||
@@ -102,14 +106,14 @@ public class DomainSchemaBuilder {
|
||||
domainSchema.getDimensions().addAll(dimensions);
|
||||
domainSchema.getDimensionValues().addAll(dimensionValues);
|
||||
|
||||
|
||||
Entity entity = resp.getEntity();
|
||||
if (Objects.nonNull(entity)) {
|
||||
SchemaElement entityElement = new SchemaElement();
|
||||
|
||||
if (!CollectionUtils.isEmpty(entity.getNames()) && Objects.nonNull(entity.getEntityId())) {
|
||||
Map<Long, SchemaElement> idAndDimPair = dimensions.stream()
|
||||
.collect(Collectors.toMap(SchemaElement::getId, schemaElement -> schemaElement,(k1,k2)->k2));
|
||||
.collect(
|
||||
Collectors.toMap(SchemaElement::getId, schemaElement -> schemaElement, (k1, k2) -> k2));
|
||||
if (idAndDimPair.containsKey(entity.getEntityId())) {
|
||||
entityElement = idAndDimPair.get(entity.getEntityId());
|
||||
entityElement.setType(SchemaElementType.ENTITY);
|
||||
@@ -1,5 +1,10 @@
|
||||
package com.tencent.supersonic.knowledge.semantic;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.cache.Cache;
|
||||
@@ -8,29 +13,34 @@ import com.google.gson.Gson;
|
||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.ResultData;
|
||||
import com.tencent.supersonic.common.pojo.ReturnCode;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.exception.CommonException;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||
import com.tencent.supersonic.common.util.ThreadContext;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
|
||||
import com.tencent.supersonic.semantic.api.model.response.*;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.common.pojo.exception.CommonException;
|
||||
import com.tencent.supersonic.common.pojo.ResultData;
|
||||
import com.tencent.supersonic.common.pojo.ReturnCode;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.*;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
@@ -40,17 +50,14 @@ import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.util.UriComponentsBuilder;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.*;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
|
||||
|
||||
@Slf4j
|
||||
public class RemoteSemanticLayer extends BaseSemanticLayer {
|
||||
@Autowired
|
||||
|
||||
private S2ThreadContext s2ThreadContext;
|
||||
@Autowired
|
||||
|
||||
private AuthenticationConfig authenticationConfig;
|
||||
|
||||
private static final Cache<String, List<DomainSchemaResp>> domainSchemaCache =
|
||||
private static final Cache<String, List<ModelSchemaResp>> domainSchemaCache =
|
||||
CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build();
|
||||
private ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>> structTypeRef =
|
||||
new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
|
||||
@@ -109,7 +116,8 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
|
||||
throw new CommonException(responseBody.getCode(), responseBody.getMsg());
|
||||
}
|
||||
|
||||
public List<DomainSchemaResp> doFetchDomainSchema(List<Long> ids) {
|
||||
@Override
|
||||
public List<ModelSchemaResp> doFetchModelSchema(List<Long> ids) {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.set(UserConstants.INTERNAL, TRUE_LOWER);
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
@@ -117,46 +125,52 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
|
||||
URI requestUrl = UriComponentsBuilder.fromHttpUrl(
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainSchemaPath()).build()
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchModelSchemaPath()).build()
|
||||
.encode().toUri();
|
||||
DomainSchemaFilterReq filter = new DomainSchemaFilterReq();
|
||||
filter.setDomainIds(ids);
|
||||
ParameterizedTypeReference<ResultData<List<DomainSchemaResp>>> responseTypeRef =
|
||||
new ParameterizedTypeReference<ResultData<List<DomainSchemaResp>>>() {
|
||||
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
|
||||
filter.setModelIds(ids);
|
||||
ParameterizedTypeReference<ResultData<List<ModelSchemaResp>>> responseTypeRef =
|
||||
new ParameterizedTypeReference<ResultData<List<ModelSchemaResp>>>() {
|
||||
};
|
||||
|
||||
HttpEntity<String> entity = new HttpEntity<>(JSON.toJSONString(filter), headers);
|
||||
|
||||
try {
|
||||
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
|
||||
ResponseEntity<ResultData<List<DomainSchemaResp>>> responseEntity = restTemplate.exchange(
|
||||
ResponseEntity<ResultData<List<ModelSchemaResp>>> responseEntity = restTemplate.exchange(
|
||||
requestUrl, HttpMethod.POST, entity, responseTypeRef);
|
||||
ResultData<List<DomainSchemaResp>> responseBody = responseEntity.getBody();
|
||||
log.debug("ApiResponse<fetchDomainSchema> responseBody:{}", responseBody);
|
||||
ResultData<List<ModelSchemaResp>> responseBody = responseEntity.getBody();
|
||||
log.debug("ApiResponse<fetchModelSchema> responseBody:{}", responseBody);
|
||||
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
||||
List<DomainSchemaResp> data = responseBody.getData();
|
||||
List<ModelSchemaResp> data = responseBody.getData();
|
||||
return data;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("fetchDomainSchema interface error", e);
|
||||
throw new RuntimeException("fetchModelSchema interface error", e);
|
||||
}
|
||||
throw new RuntimeException("fetchDomainSchema interface error");
|
||||
throw new RuntimeException("fetchModelSchema interface error");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForViewer() {
|
||||
public List<DomainResp> getDomainList(User user) {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
Object domainDescListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainViewListPath(), null, HttpMethod.GET);
|
||||
List<DomainResp> domainDescList = JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class);
|
||||
return domainDescList;
|
||||
Object domainDescListObject = fetchHttpResult(
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainListPath(), null,
|
||||
HttpMethod.GET);
|
||||
return JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForAdmin() {
|
||||
public List<ModelResp> getModelList(AuthType authType, Long domainId, User user) {
|
||||
if (domainId == null) {
|
||||
domainId = 0L;
|
||||
}
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
Object domainDescListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainListPath(), null, HttpMethod.GET);
|
||||
List<DomainResp> domainDescList = JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class);
|
||||
return domainDescList;
|
||||
String url = String.format("%s?domainId=%s&authType=%s",
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchModelListPath()
|
||||
, domainId, authType.toString());
|
||||
Object domainDescListObject = fetchHttpResult(url, null, HttpMethod.GET);
|
||||
return JsonUtil.toList(JsonUtil.toString(domainDescListObject), ModelResp.class);
|
||||
}
|
||||
|
||||
public Object fetchHttpResult(String url, String bodyJson, HttpMethod httpMethod) {
|
||||
@@ -173,15 +187,15 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
|
||||
ResponseEntity<ResultData<Object>> responseEntity = restTemplate.exchange(requestUrl,
|
||||
httpMethod, entity, responseTypeRef);
|
||||
ResultData<Object> responseBody = responseEntity.getBody();
|
||||
log.debug("ApiResponse<fetchDomainSchema> responseBody:{}", responseBody);
|
||||
log.debug("ApiResponse<fetchModelSchema> responseBody:{}", responseBody);
|
||||
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
||||
Object data = responseBody.getData();
|
||||
return data;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("fetchDomainSchema interface error", e);
|
||||
throw new RuntimeException("fetchModelSchema interface error", e);
|
||||
}
|
||||
throw new RuntimeException("fetchDomainSchema interface error");
|
||||
throw new RuntimeException("fetchModelSchema interface error");
|
||||
}
|
||||
|
||||
public void fillToken(HttpHeaders headers) {
|
||||
@@ -203,7 +217,9 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
|
||||
String body = JsonUtil.toString(pageMetricCmd);
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
log.info("url:{}", defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath());
|
||||
Object dimensionListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath(), body, HttpMethod.POST);
|
||||
Object dimensionListObject = fetchHttpResult(
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath(), body,
|
||||
HttpMethod.POST);
|
||||
LinkedHashMap map = (LinkedHashMap) dimensionListObject;
|
||||
PageInfo<Object> metricDescObjectPageInfo = generatePageInfo(map);
|
||||
PageInfo<MetricResp> metricDescPageInfo = new PageInfo<>();
|
||||
@@ -216,7 +232,9 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
|
||||
public PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd) {
|
||||
String body = JsonUtil.toString(pageDimensionCmd);
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
Object dimensionListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDimensionPagePath(), body, HttpMethod.POST);
|
||||
Object dimensionListObject = fetchHttpResult(
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDimensionPagePath(), body,
|
||||
HttpMethod.POST);
|
||||
LinkedHashMap map = (LinkedHashMap) dimensionListObject;
|
||||
PageInfo<Object> dimensionDescObjectPageInfo = generatePageInfo(map);
|
||||
PageInfo<DimensionResp> dimensionDescPageInfo = new PageInfo<>();
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.tencent.supersonic.knowledge.service;
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface KnowledgeService {
|
||||
|
||||
@@ -3,10 +3,8 @@ package com.tencent.supersonic.knowledge.service;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
||||
@@ -4,14 +4,13 @@ import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.knowledge.utils.ComponentFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class SchemaService {
|
||||
@@ -28,13 +27,13 @@ public class SchemaService {
|
||||
@Override
|
||||
public SemanticSchema load(String key) {
|
||||
log.info("load getDomainSchemaInfo cache [{}]", key);
|
||||
return new SemanticSchema(semanticLayer.getDomainSchema());
|
||||
return new SemanticSchema(semanticLayer.getModelSchema());
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
public DomainSchema getDomainSchema(Long id) {
|
||||
return semanticLayer.getDomainSchema(id, true);
|
||||
public ModelSchema getModelSchema(Long id) {
|
||||
return semanticLayer.getModelSchema(id, true);
|
||||
}
|
||||
|
||||
public SemanticSchema getSemanticSchema() {
|
||||
|
||||
@@ -5,9 +5,9 @@ import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
|
||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictionaryAttributeUtil;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -15,8 +15,6 @@ import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictionaryAttributeUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -6,11 +6,9 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory;
|
||||
|
||||
import com.tencent.supersonic.knowledge.utils.ComponentFactory;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.tencent.supersonic.knowledge.utils.ComponentFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -23,13 +21,13 @@ public class WordService {
|
||||
|
||||
public List<DictWord> getAllDictWords() {
|
||||
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
SemanticSchema semanticSchema = new SemanticSchema(semanticLayer.getDomainSchema());
|
||||
SemanticSchema semanticSchema = new SemanticSchema(semanticLayer.getModelSchema());
|
||||
|
||||
List<DictWord> words = new ArrayList<>();
|
||||
|
||||
addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words);
|
||||
addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words);
|
||||
addWordsByType(DictWordType.DOMAIN, semanticSchema.getDomains(), words);
|
||||
addWordsByType(DictWordType.DOMAIN, semanticSchema.getModels(), words);
|
||||
addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words);
|
||||
addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words);
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
package com.tencent.supersonic.knowledge.utils;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||
|
||||
public class ComponentFactory {
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueInfo;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Date;
|
||||
@@ -39,7 +38,7 @@ public class DictTaskConverter {
|
||||
|
||||
public static DictConfig dictConfPO2Config(DictConfDO dictConfDO) {
|
||||
DictConfig dictConfig = new DictConfig();
|
||||
dictConfig.setDomainId(dictConfDO.getDomainId());
|
||||
dictConfig.setModelId(dictConfDO.getModelId());
|
||||
List<DimValueInfo> dimValueInfos = JsonUtil.toList(dictConfDO.getDimValueInfos(), DimValueInfo.class);
|
||||
dictConfig.setDimValueInfoList(dimValueInfos);
|
||||
return dictConfig;
|
||||
|
||||
@@ -7,7 +7,6 @@ import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
|
||||
@@ -7,18 +7,17 @@ import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||
import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
|
||||
import com.hankcs.hanlp.seg.Segment;
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.dictionary.HadoopFileIOAdapter;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MultiCustomDictionary;
|
||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.HadoopFileIOAdapter;
|
||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MultiCustomDictionary;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
|
||||
@@ -8,7 +8,6 @@ import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
<mapper namespace="com.tencent.supersonic.knowledge.persistence.mapper.DictConfMapper">
|
||||
|
||||
<resultMap id="DictConfPO"
|
||||
type="com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO">
|
||||
type="com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO">
|
||||
<id column="id" property="id"/>
|
||||
<result column="domain_id" property="domainId"/>
|
||||
<result column="model_id" property="modelId"/>
|
||||
<result column="dim_value_infos" property="dimValueInfos"/>
|
||||
<result column="created_at" property="createdAt"/>
|
||||
<result column="updated_at" property="updatedAt"/>
|
||||
@@ -18,35 +18,35 @@
|
||||
<insert id="createDictConf">
|
||||
insert into s2_dictionary
|
||||
(`domain_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
|
||||
values
|
||||
(#{domainId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
|
||||
values (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy},
|
||||
#{updatedBy})
|
||||
</insert>
|
||||
|
||||
<insert id="upsertDictInfo">
|
||||
insert into s2_dictionary
|
||||
(`domain_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
|
||||
values
|
||||
(#{domainId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
|
||||
on duplicate key update
|
||||
dim_value_infos = #{dimValueInfos},
|
||||
updated_at = #{updatedAt},
|
||||
updated_by = #{updatedBy}
|
||||
insert into s2_dictionary
|
||||
(`model_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
|
||||
values (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy},
|
||||
#{updatedBy}) on duplicate key
|
||||
update
|
||||
dim_value_infos = #{dimValueInfos},
|
||||
updated_at = #{updatedAt},
|
||||
updated_by = #{updatedBy}
|
||||
</insert>
|
||||
|
||||
<update id="editDictConf">
|
||||
update s2_dictionary
|
||||
set dim_value_infos = #{dimValueInfos},
|
||||
updated_at = #{updatedAt},
|
||||
updated_by = #{updatedBy}
|
||||
where domain_id = #{domainId}
|
||||
and status = 0
|
||||
updated_at = #{updatedAt},
|
||||
updated_by = #{updatedBy}
|
||||
where model_id = #{modelId}
|
||||
and status = 0
|
||||
</update>
|
||||
|
||||
<select id="getDictInfoByDomainId" resultMap="DictConfPO">
|
||||
<select id="getDictInfoByModelId" resultMap="DictConfPO">
|
||||
select *
|
||||
from s2_dictionary
|
||||
where domain_id = #{domainId}
|
||||
and status = 0
|
||||
where model_id = #{modelId}
|
||||
and status = 0
|
||||
</select>
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<mapper namespace="com.tencent.supersonic.knowledge.persistence.mapper.DictTaskMapper">
|
||||
|
||||
<resultMap id="DimValueDictTaskPO"
|
||||
type="com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO">
|
||||
type="com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO">
|
||||
<id column="id" property="id"/>
|
||||
<result column="name" property="name"/>
|
||||
<result column="description" property="description"/>
|
||||
@@ -21,8 +21,8 @@
|
||||
<insert id="createDimValueTask">
|
||||
insert into s2_dictionary_task
|
||||
(`name`, description, command, command_md5, status, created_by, progress, elapsed_ms)
|
||||
values
|
||||
(#{name}, #{description}, #{command}, #{commandMd5}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
|
||||
values (#{name}, #{description}, #{command}, #{commandMd5}, #{status}, #{createdBy},
|
||||
#{progress}, #{elapsedMs})
|
||||
</insert>
|
||||
|
||||
<update id="updateTaskStatus">
|
||||
@@ -54,7 +54,7 @@
|
||||
and id >= #{id}
|
||||
</if>
|
||||
<if test="name != null and name !=''">
|
||||
and `name` like "%"#{name}"%"
|
||||
and `name` like "%"#{name}"%"
|
||||
</if>
|
||||
<if test="createdBy != null and createdBy !=''">
|
||||
and created_by = #{createdBy}
|
||||
|
||||
@@ -1,15 +1,40 @@
|
||||
CREATE TABLE IF NOT EXISTS `s2_dictionary` (
|
||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`item_id` bigint(20) DEFAULT NULL COMMENT '对应维度id、指标id等',
|
||||
`type` varchar(50) DEFAULT NULL COMMENT '对应维度、指标等',
|
||||
`black_list` mediumtext COMMENT '字典黑名单',
|
||||
`white_list` mediumtext COMMENT '字典白名单',
|
||||
`rule_list` mediumtext COMMENT '字典规则',
|
||||
`is_dict_Info` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-开启写入字典,0-不开启',
|
||||
`created_at` datetime NOT NULL COMMENT '创建时间',
|
||||
`updated_at` datetime NOT NULL COMMENT '更新时间',
|
||||
`created_by` varchar(100) NOT NULL COMMENT '创建人',
|
||||
`updated_by` varchar(100) DEFAULT NULL COMMENT '更新人',
|
||||
`is_deleted` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-删除,0-可用',
|
||||
PRIMARY KEY (`id`)
|
||||
) COMMENT='字典配置信息表'
|
||||
CREATE TABLE IF NOT EXISTS `s2_dictionary`
|
||||
(
|
||||
`id` bigint
|
||||
(
|
||||
20
|
||||
) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`item_id` bigint
|
||||
(
|
||||
20
|
||||
) DEFAULT NULL COMMENT '对应维度id、指标id等',
|
||||
`type` varchar
|
||||
(
|
||||
50
|
||||
) DEFAULT NULL COMMENT '对应维度、指标等',
|
||||
`black_list` mediumtext COMMENT '字典黑名单',
|
||||
`white_list` mediumtext COMMENT '字典白名单',
|
||||
`rule_list` mediumtext COMMENT '字典规则',
|
||||
`is_dict_Info` tinyint
|
||||
(
|
||||
1
|
||||
) NOT NULL DEFAULT '0' COMMENT '1-开启写入字典,0-不开启',
|
||||
`created_at` datetime NOT NULL COMMENT '创建时间',
|
||||
`updated_at` datetime NOT NULL COMMENT '更新时间',
|
||||
`created_by` varchar
|
||||
(
|
||||
100
|
||||
) NOT NULL COMMENT '创建人',
|
||||
`updated_by` varchar
|
||||
(
|
||||
100
|
||||
) DEFAULT NULL COMMENT '更新人',
|
||||
`is_deleted` tinyint
|
||||
(
|
||||
1
|
||||
) NOT NULL DEFAULT '0' COMMENT '1-删除,0-可用',
|
||||
PRIMARY KEY
|
||||
(
|
||||
`id`
|
||||
)
|
||||
) COMMENT='字典配置信息表'
|
||||
|
||||
@@ -1,11 +1,33 @@
|
||||
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
|
||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(255) NOT NULL COMMENT '任务名称',
|
||||
`description` varchar(255) NOT NULL COMMENT '任务描述',
|
||||
`command` mediumtext NOT NULL COMMENT '任务请求参数',
|
||||
`status` int(10) NOT NULL COMMENT '任务最终运行状态',
|
||||
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`created_by` varchar(100) NOT NULL COMMENT '创建人',
|
||||
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时',
|
||||
PRIMARY KEY (`id`)
|
||||
)COMMENT='字典任务信息表'
|
||||
CREATE TABLE IF NOT EXISTS `s2_dictionary_task`
|
||||
(
|
||||
`id` bigint
|
||||
(
|
||||
20
|
||||
) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar
|
||||
(
|
||||
255
|
||||
) NOT NULL COMMENT '任务名称',
|
||||
`description` varchar
|
||||
(
|
||||
255
|
||||
) NOT NULL COMMENT '任务描述',
|
||||
`command` mediumtext NOT NULL COMMENT '任务请求参数',
|
||||
`status` int
|
||||
(
|
||||
10
|
||||
) NOT NULL COMMENT '任务最终运行状态',
|
||||
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`created_by` varchar
|
||||
(
|
||||
100
|
||||
) NOT NULL COMMENT '创建人',
|
||||
`elapsed_ms` bigint
|
||||
(
|
||||
10
|
||||
) DEFAULT NULL COMMENT '任务耗时',
|
||||
PRIMARY KEY
|
||||
(
|
||||
`id`
|
||||
)
|
||||
) COMMENT='字典任务信息表'
|
||||
|
||||
Reference in New Issue
Block a user