topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Put term into dict and let it can be mapped by mapper (#1002)

Browse Source
This commit is contained in:
LXW
2024-05-16 10:15:04 +08:00
committed by GitHub
parent 21af74c674
commit 55c625a915
34 changed files with 292 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.common.pojo.enums;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import java.util.Objects;
/***
* nature type
@@ -23,6 +23,8 @@ public enum DictWordType {
TAG("tag"),
TERM("term"),
SUFFIX("suffix");
public static final String NATURE_SPILT = "_";

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import lombok.Data;
@Data
public class DataSetSchema {
@@ -12,6 +12,7 @@ public class DataSetSchema {
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<SchemaElement> tags = new HashSet<>();
private Set<SchemaElement> dimensionValues = new HashSet<>();
private Set<SchemaElement> terms = new HashSet<>();
private SchemaElement entity = new SchemaElement();
private QueryConfig queryConfig;
@@ -37,6 +38,9 @@ public class DataSetSchema {
case TAG:
element = tags.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case TERM:
element = terms.stream().filter(e -> e.getId() == elementID).findFirst();
break;
default:
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java
View File

@@ -8,5 +8,6 @@ public enum SchemaElementType {
ENTITY,
ID,
DATE,
TAG
TAG,
TERM
}

9
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java
View File

@@ -44,6 +44,9 @@ public class SemanticSchema implements Serializable {
case TAG:
element = getElementsById(elementID, getTags());
break;
case TERM:
element = getElementsById(elementID, getTerms());
break;
default:
}
@@ -118,6 +121,12 @@ public class SemanticSchema implements Serializable {
return tags;
}
public List<SchemaElement> getTerms() {
List<SchemaElement> terms = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> terms.addAll(d.getTerms()));
return terms;
}
private List<SchemaElement> getElementsByDataSetId(Long dataSetId, List<SchemaElement> elements) {
return elements.stream()
.filter(schemaElement -> dataSetId.equals(schemaElement.getDataSet()))

16
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Term.java
View File

@@ -1,16 +0,0 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class Term {
private String name;
private String description;
private List<String> similarTerms = Lists.newArrayList();
}

11
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TermSetReq.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TermReq.java
View File

@@ -2,18 +2,23 @@ package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.headless.api.pojo.Term;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.List;
@Data
public class TermSetReq extends RecordInfo {
public class TermReq extends RecordInfo {
private Long id;
@NotNull(message = "主题域ID不可为空")
private Long domainId;
private List<Term> terms = Lists.newArrayList();
private String name;
private String description;
private List<String> alias = Lists.newArrayList();
}

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java
View File

@@ -16,6 +16,7 @@ public class DataSetSchemaResp extends DataSetResp {
private List<MetricSchemaResp> metrics = Lists.newArrayList();
private List<DimSchemaResp> dimensions = Lists.newArrayList();
private List<ModelResp> modelResps = Lists.newArrayList();
private List<TermResp> termResps = Lists.newArrayList();
public DimSchemaResp getPrimaryKey() {
for (ModelResp modelResp : modelResps) {

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.Term;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import lombok.Data;
import java.util.HashMap;
@@ -14,6 +14,6 @@ public class MapInfoResp {
private Map<String, DataSetMapInfo> dataSetMapInfo = new HashMap<>();
private Map<String, List<Term>> terms = new HashMap<>();
private Map<String, List<SchemaElementMatch>> terms = new HashMap<>();
}

16
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TermSetResp.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TermResp.java
View File

@@ -2,23 +2,27 @@ package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.headless.api.pojo.Term;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotNull;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class TermSetResp extends RecordInfo {
public class TermResp extends RecordInfo {
private Long id;
@NotNull(message = "主题域ID不可为空")
private Long domainId;
private List<Term> terms = Lists.newArrayList();
private String name;
private String description;
private List<String> alias = Lists.newArrayList();
public TermSetResp(Long domainId) {
this.domainId = domainId;
}
}

44
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/TermWordBuilder.java Normal file
View File

@@ -0,0 +1,44 @@
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* Metric DictWord
*/
@Service
public class TermWordBuilder extends BaseWordWithAliasBuilder {
@Override
public List<DictWord> doGet(String word, SchemaElement schemaElement) {
List<DictWord> result = Lists.newArrayList();
result.add(getOneWordNature(word, schemaElement, false));
result.addAll(getOneWordNatureAlias(schemaElement, false));
String reverseWord = StringUtils.reverse(word);
if (!word.equalsIgnoreCase(reverseWord)) {
result.add(getOneWordNature(reverseWord, schemaElement, true));
}
return result;
}
public DictWord getOneWordNature(String word, SchemaElement schemaElement, boolean isSuffix) {
DictWord dictWord = new DictWord();
dictWord.setWord(word);
Long dataSet = schemaElement.getDataSet();
String nature = DictWordType.NATURE_SPILT + dataSet + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.TERM.getType();
if (isSuffix) {
nature = DictWordType.NATURE_SPILT + dataSet + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.SUFFIX.getType() + DictWordType.TERM.getType();
}
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
return dictWord;
}
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/WordBuilderFactory.java
View File

@@ -18,6 +18,7 @@ public class WordBuilderFactory {
wordNatures.put(DictWordType.DATASET, new ModelWordBuilder());
wordNatures.put(DictWordType.ENTITY, new EntityWordBuilder());
wordNatures.put(DictWordType.VALUE, new ValueWordBuilder());
wordNatures.put(DictWordType.TERM, new TermWordBuilder());
}
public static BaseWordBuilder get(DictWordType strategyType) {

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/helper/HanlpHelper.java
View File

@@ -214,17 +214,17 @@ public class HanlpHelper {
}
}
public static List<S2Term> getTerms(String text, Map<Long, List<Long>> modelIdToViewIds) {
public static List<S2Term> getTerms(String text, Map<Long, List<Long>> modelIdToDataSetIds) {
return getSegment().seg(text.toLowerCase()).stream()
.filter(term -> term.getNature().startsWith(DictWordType.NATURE_SPILT))
.map(term -> transform2ApiTerm(term, modelIdToViewIds))
.map(term -> transform2ApiTerm(term, modelIdToDataSetIds))
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
public static List<S2Term> transform2ApiTerm(Term term, Map<Long, List<Long>> modelIdToViewIds) {
public static List<S2Term> transform2ApiTerm(Term term, Map<Long, List<Long>> modelIdToDataSetIds) {
List<S2Term> s2Terms = Lists.newArrayList();
List<String> natures = NatureHelper.changeModel2DataSet(String.valueOf(term.getNature()), modelIdToViewIds);
List<String> natures = NatureHelper.changeModel2DataSet(String.valueOf(term.getNature()), modelIdToDataSetIds);
for (String nature : natures) {
S2Term s2Term = new S2Term();
BeanUtils.copyProperties(term, s2Term);

16
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/helper/NatureHelper.java
View File

@@ -6,6 +6,10 @@ import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.chat.knowledge.DataSetInfoStat;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -14,9 +18,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
/**
* nature parse helper
@@ -46,6 +47,9 @@ public class NatureHelper {
case VALUE:
result = SchemaElementType.VALUE;
break;
case TERM:
result = SchemaElementType.TERM;
break;
default:
break;
}
@@ -108,6 +112,10 @@ public class NatureHelper {
}
public static List<String> changeModel2DataSet(String nature, Map<Long, List<Long>> modelIdToDataSetIds) {
//term prefix id is dataSetId, no need to transform
if (SchemaElementType.TERM.equals(NatureHelper.convertToElementType(nature))) {
return Lists.newArrayList(nature);
}
Long modelId = getModelId(nature);
List<Long> dataSetIds = modelIdToDataSetIds.get(modelId);
if (CollectionUtils.isEmpty(dataSetIds)) {
@@ -129,7 +137,7 @@ public class NatureHelper {
return false;
}
return !nature.endsWith(DictWordType.METRIC.getType()) && !nature.endsWith(
DictWordType.DIMENSION.getType())
DictWordType.DIMENSION.getType()) && !nature.endsWith(DictWordType.TERM.getType())
&& StringUtils.isNumeric(split[1]);
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMapper.java
View File

@@ -111,6 +111,10 @@ public abstract class BaseMapper implements SchemaMapper {
if (!existElement.equals(newElement)) {
return false;
}
if (SchemaElementType.TERM.equals(existElement.getType())
&& SchemaElementType.TERM.equals(newElement.getType())) {
return false;
}
if (SchemaElementType.VALUE.equals(newElement.getType())) {
return existElementMatch.getWord().equalsIgnoreCase(newElementMatch.getWord());
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java
View File

@@ -46,7 +46,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
return null;
}
log.debug("retryCount:{},terms:{},,detectModelIds:{}", terms, detectDataSetIds);
log.debug("terms:{},detectModelIds:{}", terms, detectDataSetIds);
List<HanlpMapResult> detects = detect(queryContext, terms, detectDataSetIds);
Map<MatchText, List<HanlpMapResult>> result = new HashMap<>();

7
headless/server/pom.xml
View File

@@ -118,13 +118,6 @@
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>headless-api</artifactId>
<version>0.9.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermSetDO.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermDO.java
View File

@@ -4,18 +4,23 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_term_set")
public class TermSetDO {
@TableName("s2_term")
public class TermDO {
@TableId(value = "id", type = IdType.AUTO)
private Long id;
private Long domainId;
private String terms;
private String name;
private String description;
private String alias;
private Date createdAt;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermSetMapper.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermMapper.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.TermSetDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.TermDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TermSetMapper extends BaseMapper<TermSetDO> {
public interface TermMapper extends BaseMapper<TermDO> {
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DomainController.java
View File

@@ -32,21 +32,19 @@ public class DomainController {
}
@PostMapping("/createDomain")
public Boolean createDomain(@RequestBody DomainReq domainReq,
public DomainResp createDomain(@RequestBody DomainReq domainReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
domainService.createDomain(domainReq, user);
return true;
return domainService.createDomain(domainReq, user);
}
@PostMapping("/updateDomain")
public Boolean updateDomain(@RequestBody DomainUpdateReq domainUpdateReq,
public DomainResp updateDomain(@RequestBody DomainUpdateReq domainUpdateReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
domainService.updateDomain(domainUpdateReq, user);
return true;
return domainService.updateDomain(domainUpdateReq, user);
}
@DeleteMapping("/deleteDomain/{domainId}")

21
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TermController.java
View File

@@ -2,11 +2,13 @@ package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.TermSetReq;
import com.tencent.supersonic.headless.api.pojo.response.TermSetResp;
import com.tencent.supersonic.headless.api.pojo.request.TermReq;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import com.tencent.supersonic.headless.server.service.TermService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -15,6 +17,7 @@ import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/term")
@@ -24,17 +27,23 @@ public class TermController {
private TermService termService;
@PostMapping("/saveOrUpdate")
public boolean saveOrUpdate(@RequestBody TermSetReq termSetReq,
public boolean saveOrUpdate(@RequestBody TermReq termReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
termService.saveOrUpdate(termSetReq, user);
termService.saveOrUpdate(termReq, user);
return true;
}
@GetMapping
public TermSetResp getTermSet(@RequestParam("domainId") Long domainId) {
return termService.getTermSet(domainId);
public List<TermResp> getTerms(@RequestParam("domainId") Long domainId) {
return termService.getTerms(domainId);
}
@DeleteMapping("/{id}")
public boolean delete(@PathVariable("id") Long id) {
termService.delete(id);
return true;
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DomainService.java
View File

@@ -16,9 +16,9 @@ public interface DomainService {
Map<Long, String> getDomainFullPath();
void createDomain(DomainReq domainReq, User user);
DomainResp createDomain(DomainReq domainReq, User user);
void updateDomain(DomainUpdateReq domainUpdateReq, User user);
DomainResp updateDomain(DomainUpdateReq domainUpdateReq, User user);
void deleteDomain(Long id);

13
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TermService.java
View File

@@ -2,9 +2,8 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.Term;
import com.tencent.supersonic.headless.api.pojo.request.TermSetReq;
import com.tencent.supersonic.headless.api.pojo.response.TermSetResp;
import com.tencent.supersonic.headless.api.pojo.request.TermReq;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import java.util.List;
import java.util.Map;
@@ -12,10 +11,12 @@ import java.util.Set;
public interface TermService {
void saveOrUpdate(TermSetReq termSetReq, User user);
void saveOrUpdate(TermReq termSetReq, User user);
TermSetResp getTermSet(Long domainId);
void delete(Long id);
Map<Long, List<Term>> getTermSets(Set<Long> domainIds);
List<TermResp> getTerms(Long domainId);
Map<Long, List<TermResp>> getTermSets(Set<Long> domainIds);
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java
View File

@@ -57,18 +57,20 @@ public class DomainServiceImpl implements DomainService {
}
@Override
public void createDomain(DomainReq domainReq, User user) {
public DomainResp createDomain(DomainReq domainReq, User user) {
DomainDO domainDO = DomainConvert.convert(domainReq, user);
domainDO.setStatus(StatusEnum.ONLINE.getCode());
domainRepository.createDomain(domainDO);
return DomainConvert.convert(domainDO);
}
@Override
public void updateDomain(DomainUpdateReq domainUpdateReq, User user) {
public DomainResp updateDomain(DomainUpdateReq domainUpdateReq, User user) {
domainUpdateReq.updatedBy(user.getName());
DomainDO domainDO = getDomainDO(domainUpdateReq.getId());
BeanMapper.mapper(domainUpdateReq, domainDO);
domainRepository.updateDomain(domainDO);
return DomainConvert.convert(domainDO);
}
@Override

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
View File

@@ -8,7 +8,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.Term;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
@@ -20,7 +19,6 @@ import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import com.tencent.supersonic.headless.server.service.TermService;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -48,9 +46,6 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
@Autowired
private SemanticService semanticService;
@Autowired
private TermService termService;
@Override
public MapInfoResp getMapMeta(QueryMapReq queryMapReq) {
@@ -77,7 +72,7 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(dataSetList, dataSetMap));
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
return mapInfoResp;
}
@@ -109,7 +104,10 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue();
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch ->
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
@@ -150,17 +148,21 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
return result;
}
private Map<String, List<Term>> getTerms(List<DataSetResp> dataSets, Map<Long, DataSetResp> dataSetNameMap) {
Set<Long> domainIds = dataSets.stream().map(DataSetResp::getDomainId).collect(Collectors.toSet());
Map<Long, Long> dataSetDomainIdMap = dataSets.stream()
.collect(Collectors.toMap(DataSetResp::getId, DataSetResp::getDomainId));
Map<Long, List<Term>> domainTermSetMap = termService.getTermSets(domainIds);
Map<String, List<Term>> dataSetTermSetMap = new HashMap<>();
for (DataSetResp dataSet : dataSets) {
dataSetTermSetMap.put(dataSetNameMap.get(dataSet.getId()).getName(),
domainTermSetMap.get(dataSetDomainIdMap.get(dataSet.getId())));
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return dataSetTermSetMap;
return termMap;
}
/***

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -33,6 +33,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
@@ -44,6 +45,7 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TermService;
import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -60,6 +62,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -86,6 +89,7 @@ public class SchemaServiceImpl implements SchemaService {
private final DataSetService dataSetService;
private final ModelRelaService modelRelaService;
private final TagMetaService tagService;
private final TermService termService;
public SchemaServiceImpl(ModelService modelService,
DimensionService dimensionService,
@@ -93,7 +97,7 @@ public class SchemaServiceImpl implements SchemaService {
DomainService domainService,
DataSetService dataSetService,
ModelRelaService modelRelaService,
StatUtils statUtils, TagMetaService tagService) {
StatUtils statUtils, TagMetaService tagService, TermService termService) {
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
@@ -102,6 +106,7 @@ public class SchemaServiceImpl implements SchemaService {
this.modelRelaService = modelRelaService;
this.statUtils = statUtils;
this.tagService = tagService;
this.termService = termService;
}
@SneakyThrows
@@ -165,9 +170,12 @@ public class SchemaServiceImpl implements SchemaService {
metaFilter.setIds(filter.getDataSetIds());
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(metaFilter);
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
Set<Long> domainIds = dataSetResps.stream().map(DataSetResp::getDomainId)
.collect(Collectors.toSet());
List<Long> modelIds = dataSetRespMap.values().stream().map(DataSetResp::getAllModels)
.flatMap(Collection::stream).collect(Collectors.toList());
Map<Long, List<TermResp>> termMaps = termService.getTermSets(domainIds);
metaFilter.setModelIds(modelIds);
metaFilter.setIds(Lists.newArrayList());
@@ -198,7 +206,7 @@ public class SchemaServiceImpl implements SchemaService {
dataSetSchemaResp.setMetrics(metricSchemaResps);
dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
dataSetSchemaResp.setTermResps(termMaps.getOrDefault(dataSetResp.getDomainId(), Lists.newArrayList()));
dataSetSchemaResps.add(dataSetSchemaResp);
}
fillStaticInfo(dataSetSchemaResps);

60
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TermServiceImpl.java
View File

@@ -5,11 +5,10 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.Term;
import com.tencent.supersonic.headless.api.pojo.request.TermSetReq;
import com.tencent.supersonic.headless.api.pojo.response.TermSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.TermSetDO;
import com.tencent.supersonic.headless.server.persistence.mapper.TermSetMapper;
import com.tencent.supersonic.headless.api.pojo.request.TermReq;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.TermDO;
import com.tencent.supersonic.headless.server.persistence.mapper.TermMapper;
import com.tencent.supersonic.headless.server.service.TermService;
import org.springframework.stereotype.Service;
import java.util.List;
@@ -19,48 +18,49 @@ import java.util.stream.Collectors;
@Service
public class TermServiceImpl extends ServiceImpl<TermSetMapper, TermSetDO> implements TermService {
public class TermServiceImpl extends ServiceImpl<TermMapper, TermDO> implements TermService {
@Override
public void saveOrUpdate(TermSetReq termSetReq, User user) {
QueryWrapper<TermSetDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(TermSetDO::getDomainId, termSetReq.getDomainId());
TermSetDO termSetDO = getOne(queryWrapper);
public void saveOrUpdate(TermReq termSetReq, User user) {
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(TermDO::getId, termSetReq.getId());
TermDO termSetDO = getOne(queryWrapper);
if (termSetDO == null) {
termSetReq.createdBy(user.getName());
termSetDO = new TermSetDO();
termSetDO = new TermDO();
}
termSetReq.updatedBy(user.getName());
BeanMapper.mapper(termSetReq, termSetDO);
termSetDO.setTerms(JsonUtil.toString(termSetReq.getTerms()));
termSetDO.setAlias(JsonUtil.toString(termSetReq.getAlias()));
saveOrUpdate(termSetDO);
}
@Override
public TermSetResp getTermSet(Long domainId) {
QueryWrapper<TermSetDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(TermSetDO::getDomainId, domainId);
TermSetDO termSetDO = getOne(queryWrapper);
TermSetResp termSetResp = new TermSetResp(domainId);
if (termSetDO == null) {
return termSetResp;
}
return convert(termSetDO);
public void delete(Long id) {
removeById(id);
}
@Override
public Map<Long, List<Term>> getTermSets(Set<Long> domainIds) {
QueryWrapper<TermSetDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().in(TermSetDO::getDomainId, domainIds);
List<TermSetDO> list = list(queryWrapper);
return list.stream().map(this::convert).collect(
Collectors.toMap(TermSetResp::getDomainId, TermSetResp::getTerms, (k1, k2) -> k1));
public List<TermResp> getTerms(Long domainId) {
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(TermDO::getDomainId, domainId);
List<TermDO> termDOS = list(queryWrapper);
return termDOS.stream().map(this::convert).collect(Collectors.toList());
}
private TermSetResp convert(TermSetDO termSetDO) {
TermSetResp termSetResp = new TermSetResp();
@Override
public Map<Long, List<TermResp>> getTermSets(Set<Long> domainIds) {
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().in(TermDO::getDomainId, domainIds);
List<TermDO> list = list(queryWrapper);
return list.stream().map(this::convert).collect(
Collectors.groupingBy(TermResp::getDomainId));
}
private TermResp convert(TermDO termSetDO) {
TermResp termSetResp = new TermResp();
BeanMapper.mapper(termSetDO, termSetResp);
termSetResp.setTerms(JsonUtil.toList(termSetDO.getTerms(), Term.class));
termSetResp.setAlias(JsonUtil.toList(termSetDO.getAlias(), String.class));
return termSetResp;
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/WordService.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.builder.WordBuilderFactory;
@@ -9,6 +10,7 @@ import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
@@ -34,7 +36,7 @@ public class WordService {
addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words);
addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words);
addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words);
addWordsByType(DictWordType.TERM, semanticSchema.getTerms(), words);
return words;
}
@@ -54,6 +56,12 @@ public class WordService {
}
private List<SchemaElement> distinct(List<SchemaElement> metas) {
if (CollectionUtils.isEmpty(metas)) {
return metas;
}
if (SchemaElementType.TERM.equals(metas.get(0).getType())) {
return metas;
}
return metas.stream()
.collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1))
.values()

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java
View File

@@ -12,6 +12,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
@@ -54,6 +55,9 @@ public class DataSetSchemaBuilder {
Set<SchemaElement> dimensionValues = getDimensionValues(resp);
dataSetSchema.getDimensionValues().addAll(dimensionValues);
Set<SchemaElement> terms = getTerms(resp);
dataSetSchema.getTerms().addAll(terms);
SchemaElement entity = getEntity(resp);
if (Objects.nonNull(entity)) {
dataSetSchema.setEntity(entity);
@@ -230,6 +234,28 @@ public class DataSetSchemaBuilder {
return metrics;
}
private static Set<SchemaElement> getTerms(DataSetSchemaResp resp) {
Set<SchemaElement> terms = new HashSet<>();
for (TermResp termResp : resp.getTermResps()) {
List<String> alias = termResp.getAlias();
SchemaElement metricToAdd = SchemaElement.builder()
.dataSet(resp.getId())
.dataSetName(resp.getName())
.model(-1L)
.id(termResp.getId())
.name(termResp.getName())
.bizName(termResp.getName())
.type(SchemaElementType.TERM)
.useCnt(0L)
.alias(alias)
.description(termResp.getDescription())
.build();
terms.add(metricToAdd);
}
return terms;
}
private static List<RelatedSchemaElement> getRelateSchemaElement(MetricSchemaResp metricSchemaResp) {
RelateDimension relateDimension = metricSchemaResp.getRelateDimension();
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java
View File

@@ -10,6 +10,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
public class DomainConvert {
@@ -40,4 +41,8 @@ public class DomainConvert {
return domainResp;
}
public static DomainResp convert(DomainDO domainDO) {
return convert(domainDO, new HashMap<>());
}
}

25
launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
View File

@@ -49,6 +49,7 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.TagObjectReq;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.request.TermReq;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -58,6 +59,7 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TagObjectService;
import com.tencent.supersonic.headless.server.service.TermService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -94,9 +96,10 @@ public class ModelDemoDataLoader {
private DataSourceProperties dataSourceProperties;
@Autowired
private TagObjectService tagObjectService;
@Autowired
private TagMetaService tagMetaService;
@Autowired
private TermService termService;
public void doRun() {
try {
@@ -121,6 +124,8 @@ public class ModelDemoDataLoader {
addDataSet_2();
addAuthGroup_1();
addAuthGroup_2();
addTerm();
addTerm_1();
} catch (Exception e) {
log.error("Failed to add model demo data", e);
}
@@ -556,6 +561,24 @@ public class ModelDemoDataLoader {
dataSetService.save(dataSetReq, User.getFakeUser());
}
public void addTerm() {
TermReq termReq = new TermReq();
termReq.setName("近期");
termReq.setDescription("指近10天");
termReq.setAlias(Lists.newArrayList("近一段时间"));
termReq.setDomainId(1L);
termService.saveOrUpdate(termReq, User.getFakeUser());
}
public void addTerm_1() {
TermReq termReq = new TermReq();
termReq.setName("核心用户");
termReq.setDescription("核心用户指tom和lucy");
termReq.setAlias(Lists.newArrayList("VIP用户"));
termReq.setDomainId(1L);
termService.saveOrUpdate(termReq, User.getFakeUser());
}
public void addAuthGroup_1() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(3L);

6
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -296,10 +296,12 @@ update s2_metric set is_publish = 1;
alter table s2_dimension add column `ext` varchar(1000) DEFAULT NULL;
--20240510
CREATE TABLE IF NOT EXISTS `s2_term_set` (
CREATE TABLE IF NOT EXISTS `s2_term` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20),
`terms` varchar(1000),
`name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`alias` varchar(1000) NOT NULL ,
`created_at` datetime NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` datetime DEFAULT NULL ,

8
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -627,14 +627,16 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
);
COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
CREATE TABLE IF NOT EXISTS `s2_term_set` (
CREATE TABLE IF NOT EXISTS `s2_term` (
`id` INT NOT NULL AUTO_INCREMENT,
`domain_id` INT ,
`terms` varchar(1000),
`name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`alias` varchar(1000) NOT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_term_set IS 'term info';
COMMENT ON TABLE s2_term IS 'term info';

6
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -550,10 +550,12 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
PRIMARY KEY (`id`)
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='查询规则表';
CREATE TABLE IF NOT EXISTS `s2_term_set` (
CREATE TABLE IF NOT EXISTS `s2_term` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20),
`terms` varchar(1000),
`name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`alias` varchar(1000) NOT NULL ,
`created_at` datetime NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` datetime DEFAULT NULL ,

12
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -627,14 +627,16 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
);
COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
CREATE TABLE IF NOT EXISTS `s2_term_set` (
`id` INT NOT NULL AUTO_INCREMENT,
`domain_id` INT ,
`terms` varchar(1000),
CREATE TABLE IF NOT EXISTS `s2_term` (
`id` INT NOT NULL AUTO_INCREMENT,
`domain_id` INT ,
`name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`alias` varchar(1000) NOT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_term_set IS 'term info';
COMMENT ON TABLE s2_term IS 'term info';