topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-15 14:36:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)(chat) Add views and adapt chat and headless (#700)

Browse Source 
* (improvement)(headless)(chat) Add views and adapt chat and headless

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-30 20:43:53 +08:00
committed by GitHub
parent 31f8c1df35
commit 24b442baef
237 changed files with 3205 additions and 4479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java
View File

@@ -32,8 +32,8 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String nature = sign + dataItem.getModelId() + sign + dataItem.getId()
+ sign + dataItem.getType().getName();
String nature = sign + 1 + sign + dataItem.getId()
+ sign + dataItem.getType().name().toLowerCase();
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);

234
chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java
View File

@@ -1,256 +1,40 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_plugin")
public class PluginDO {
/**
*
*/
@TableId(type = IdType.AUTO)
private Long id;
/**
* DASHBOARD,WIDGET,URL
*/
private String type;
/**
*
*/
private String model;
private String view;
/**
*
*/
private String pattern;
/**
*
*/
private String parseMode;
/**
*
*/
private String name;
/**
*
*/
private Date createdAt;
/**
*
*/
private String createdBy;
/**
*
*/
private Date updatedAt;
/**
*
*/
private String updatedBy;
/**
*
*/
private String parseModeConfig;
/**
*
*/
private String config;
/**
*
*/
private String comment;
/**
* @return id
*/
public Long getId() {
return id;
}
/**
* @param id
*/
public void setId(Long id) {
this.id = id;
}
/**
* DASHBOARD,WIDGET,URL
*
* @return type DASHBOARD,WIDGET,URL
*/
public String getType() {
return type;
}
/**
* DASHBOARD,WIDGET,URL
*
* @param type DASHBOARD,WIDGET,URL
*/
public void setType(String type) {
this.type = type == null ? null : type.trim();
}
/**
* @return model
*/
public String getModel() {
return model;
}
/**
* @param model
*/
public void setModel(String model) {
this.model = model == null ? null : model.trim();
}
/**
* @return pattern
*/
public String getPattern() {
return pattern;
}
/**
* @param pattern
*/
public void setPattern(String pattern) {
this.pattern = pattern == null ? null : pattern.trim();
}
/**
* @return parse_mode
*/
public String getParseMode() {
return parseMode;
}
/**
* @param parseMode
*/
public void setParseMode(String parseMode) {
this.parseMode = parseMode == null ? null : parseMode.trim();
}
/**
* @return name
*/
public String getName() {
return name;
}
/**
* @param name
*/
public void setName(String name) {
this.name = name == null ? null : name.trim();
}
/**
* @return created_at
*/
public Date getCreatedAt() {
return createdAt;
}
/**
* @param createdAt
*/
public void setCreatedAt(Date createdAt) {
this.createdAt = createdAt;
}
/**
* @return created_by
*/
public String getCreatedBy() {
return createdBy;
}
/**
* @param createdBy
*/
public void setCreatedBy(String createdBy) {
this.createdBy = createdBy == null ? null : createdBy.trim();
}
/**
* @return updated_at
*/
public Date getUpdatedAt() {
return updatedAt;
}
/**
* @param updatedAt
*/
public void setUpdatedAt(Date updatedAt) {
this.updatedAt = updatedAt;
}
/**
* @return updated_by
*/
public String getUpdatedBy() {
return updatedBy;
}
/**
* @param updatedBy
*/
public void setUpdatedBy(String updatedBy) {
this.updatedBy = updatedBy == null ? null : updatedBy.trim();
}
/**
* @return parse_mode_config
*/
public String getParseModeConfig() {
return parseModeConfig;
}
/**
* @param parseModeConfig
*/
public void setParseModeConfig(String parseModeConfig) {
this.parseModeConfig = parseModeConfig == null ? null : parseModeConfig.trim();
}
/**
* @return config
*/
public String getConfig() {
return config;
}
/**
* @param config
*/
public void setConfig(String config) {
this.config = config == null ? null : config.trim();
}
/**
* @return comment
*/
public String getComment() {
return comment;
}
/**
* @param comment
*/
public void setComment(String comment) {
this.comment = comment == null ? null : comment.trim();
}
}

63
chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java
View File

@@ -1,69 +1,10 @@
package com.tencent.supersonic.chat.server.persistence.mapper;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface PluginDOMapper {
/**
*
* @mbg.generated
*/
long countByExample(PluginDOExample example);
public interface PluginDOMapper extends BaseMapper<PluginDO> {
/**
*
* @mbg.generated
*/
int deleteByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int insert(PluginDO record);
/**
*
* @mbg.generated
*/
int insertSelective(PluginDO record);
/**
*
* @mbg.generated
*/
List<PluginDO> selectByExampleWithBLOBs(PluginDOExample example);
/**
*
* @mbg.generated
*/
List<PluginDO> selectByExample(PluginDOExample example);
/**
*
* @mbg.generated
*/
PluginDO selectByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeySelective(PluginDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeyWithBLOBs(PluginDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKey(PluginDO record);
}

4
chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/PluginRepository.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.server.persistence.repository;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO;
import java.util.List;
@@ -16,7 +16,7 @@ public interface PluginRepository {
PluginDO getPlugin(Long id);
List<PluginDO> query(PluginDOExample pluginDOExample);
List<PluginDO> query(QueryWrapper<PluginDO> queryWrapper);
void deletePlugin(Long id);
}

14
chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.server.persistence.repository.impl;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO;
import com.tencent.supersonic.chat.server.persistence.mapper.PluginDOMapper;
import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository;
@@ -26,7 +26,7 @@ public class PluginRepositoryImpl implements PluginRepository {
@Override
public List<PluginDO> getPlugins() {
return pluginDOMapper.selectByExampleWithBLOBs(new PluginDOExample());
return pluginDOMapper.selectList(new QueryWrapper<>());
}
@Override
@@ -60,22 +60,22 @@ public class PluginRepositoryImpl implements PluginRepository {
@Override
public void updatePlugin(PluginDO pluginDO) {
pluginDOMapper.updateByPrimaryKeyWithBLOBs(pluginDO);
pluginDOMapper.updateById(pluginDO);
}
@Override
public PluginDO getPlugin(Long id) {
return pluginDOMapper.selectByPrimaryKey(id);
return pluginDOMapper.selectById(id);
}
@Override
public List<PluginDO> query(PluginDOExample pluginDOExample) {
return pluginDOMapper.selectByExampleWithBLOBs(pluginDOExample);
public List<PluginDO> query(QueryWrapper<PluginDO> queryWrapper) {
return pluginDOMapper.selectList(queryWrapper);
}
@Override
public void deletePlugin(Long id) {
pluginDOMapper.deleteByPrimaryKey(id);
pluginDOMapper.deleteById(id);
}
}

9
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java
View File

@@ -1,22 +1,23 @@
package com.tencent.supersonic.chat.server.processor.execute;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.service.SemanticService;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import org.springframework.util.CollectionUtils;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.util.CollectionUtils;
/**
* DimensionRecommendProcessor recommend some dimensions
@@ -33,13 +34,13 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
return;
}
SchemaElement element = semanticParseInfo.getMetrics().iterator().next();
List<SchemaElement> dimensionRecommended = getDimensions(element.getId(), element.getModel());
List<SchemaElement> dimensionRecommended = getDimensions(element.getId(), element.getView());
queryResult.setRecommendedDimensions(dimensionRecommended);
}
private List<SchemaElement> getDimensions(Long metricId, Long modelId) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
ViewSchema modelSchema = semanticService.getModelSchema(modelId);
List<Long> drillDownDimensions = Lists.newArrayList();
Set<SchemaElement> metricElements = modelSchema.getMetrics();
if (!CollectionUtils.isEmpty(metricElements)) {

11
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java
View File

@@ -14,6 +14,8 @@ import com.tencent.supersonic.common.util.embedding.Retrieval;
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
import org.springframework.util.CollectionUtils;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -21,7 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.util.CollectionUtils;
/**
* MetricRecommendProcessor fills recommended metrics based on embedding similarity.
@@ -45,7 +46,7 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor {
}
List<String> metricNames = Collections.singletonList(parseInfo.getMetrics().iterator().next().getName());
Map<String, String> filterCondition = new HashMap<>();
filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getModel().toString());
filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getView().toString());
filterCondition.put("type", SchemaElementType.METRIC.name());
RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames)
.filterCondition(filterCondition).queryEmbeddings(null).build();
@@ -70,9 +71,9 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor {
if (!metricIds.contains(Retrieval.getLongId(retrieval.getId()))) {
SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(retrieval.getMetadata()),
SchemaElement.class);
if (retrieval.getMetadata().containsKey("modelId")) {
String modelId = retrieval.getMetadata().get("modelId").toString();
schemaElement.setModel(Long.parseLong(modelId));
if (retrieval.getMetadata().containsKey("viewId")) {
String viewId = retrieval.getMetadata().get("viewId").toString();
schemaElement.setView(Long.parseLong(viewId));
}
schemaElement.setOrder(++metricOrder);
parseInfo.getMetrics().add(schemaElement);

7
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.server.processor.parse;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
@@ -10,9 +11,10 @@ import com.tencent.supersonic.chat.core.query.SemanticQuery;
import com.tencent.supersonic.chat.core.query.llm.analytics.MetricAnalyzeQuery;
import com.tencent.supersonic.chat.server.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.util.CollectionUtils;
/**
* EntityInfoProcessor fills core attributes of an entity so that
@@ -35,8 +37,9 @@ public class EntityInfoProcessor implements ParseResultProcessor {
return;
}
//1. set entity info
ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId());
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryContext.getUser());
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, viewSchema, queryContext.getUser());
if (QueryManager.isTagQuery(queryMode)
|| QueryManager.isMetricQuery(queryMode)) {
parseInfo.setEntityInfo(entityInfo);

39
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java
View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.chat.server.processor.parse;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.core.query.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.core.query.SemanticQuery;
import com.tencent.supersonic.chat.server.service.impl.SchemaService;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
@@ -18,6 +18,11 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -26,10 +31,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
/**
* ParseInfoProcessor extracts structured info from S2SQL so that
@@ -73,9 +74,9 @@ public class ParseInfoProcessor implements ParseResultProcessor {
}
//set filter
Set<Long> modelIds = parseInfo.getModel().getModelIds();
Long viewId = parseInfo.getViewId();
try {
Map<String, SchemaElement> fieldNameToElement = getNameToElement(modelIds);
Map<String, SchemaElement> fieldNameToElement = getNameToElement(viewId);
List<QueryFilter> result = getDimensionFilter(fieldNameToElement, expressions);
parseInfo.getDimensionFilters().addAll(result);
} catch (Exception e) {
@@ -87,31 +88,31 @@ public class ParseInfoProcessor implements ParseResultProcessor {
return;
}
List<String> allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL()));
Set<SchemaElement> metrics = getElements(modelIds, allFields, semanticSchema.getMetrics());
Set<SchemaElement> metrics = getElements(viewId, allFields, semanticSchema.getMetrics());
parseInfo.setMetrics(metrics);
if (QueryType.METRIC.equals(parseInfo.getQueryType())) {
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
List<String> groupByDimensions = getFieldsExceptDate(groupByFields);
parseInfo.setDimensions(getElements(modelIds, groupByDimensions, semanticSchema.getDimensions()));
parseInfo.setDimensions(getElements(viewId, groupByDimensions, semanticSchema.getDimensions()));
} else if (QueryType.TAG.equals(parseInfo.getQueryType())) {
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
List<String> selectDimensions = getFieldsExceptDate(selectFields);
parseInfo.setDimensions(getElements(modelIds, selectDimensions, semanticSchema.getDimensions()));
parseInfo.setDimensions(getElements(viewId, selectDimensions, semanticSchema.getDimensions()));
}
}
private Set<SchemaElement> getElements(Set<Long> modelIds, List<String> allFields, List<SchemaElement> elements) {
private Set<SchemaElement> getElements(Long viewId, List<String> allFields, List<SchemaElement> elements) {
return elements.stream()
.filter(schemaElement -> {
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
return modelIds.contains(schemaElement.getModel()) && allFields.contains(
return viewId.equals(schemaElement.getView()) && allFields.contains(
schemaElement.getName());
}
Set<String> allFieldsSet = new HashSet<>(allFields);
Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
List<String> intersection = allFieldsSet.stream()
.filter(aliasSet::contains).collect(Collectors.toList());
return modelIds.contains(schemaElement.getModel()) && (allFields.contains(
return viewId.equals(schemaElement.getView()) && (allFields.contains(
schemaElement.getName()) || !CollectionUtils.isEmpty(intersection));
}
).collect(Collectors.toSet());
@@ -193,10 +194,10 @@ public class ParseInfoProcessor implements ParseResultProcessor {
return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue());
}
protected Map<String, SchemaElement> getNameToElement(Set<Long> modelIds) {
protected Map<String, SchemaElement> getNameToElement(Long viewId) {
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
List<SchemaElement> dimensions = semanticSchema.getDimensions(modelIds);
List<SchemaElement> metrics = semanticSchema.getMetrics(modelIds);
List<SchemaElement> dimensions = semanticSchema.getDimensions(viewId);
List<SchemaElement> metrics = semanticSchema.getMetrics(viewId);
List<SchemaElement> allElements = Lists.newArrayList();
allElements.addAll(dimensions);
@@ -218,4 +219,4 @@ public class ParseInfoProcessor implements ParseResultProcessor {
(value1, value2) -> value2));
}
}
}

32
chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java
View File

@@ -3,21 +3,20 @@ package com.tencent.supersonic.chat.server.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.server.service.ConfigService;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -76,21 +75,6 @@ public class ChatConfigController {
return configService.getAllChatRichConfig();
}
@GetMapping("/modelList/{domainId}")
public List<ModelResp> getModelList(@PathVariable("domainId") Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return semanticInterpreter.getModelList(AuthType.ADMIN, domainId, user);
}
@GetMapping("/modelList")
public List<ModelResp> getModelList(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return semanticInterpreter.getModelList(AuthType.ADMIN, null, user);
}
@GetMapping("/domainList")
public List<DomainResp> getDomainList(HttpServletRequest request,
HttpServletResponse response) {
@@ -98,17 +82,13 @@ public class ChatConfigController {
return semanticInterpreter.getDomainList(user);
}
@GetMapping("/modelList/view")
public List<ModelResp> getModelListVisible(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return semanticInterpreter.getModelList(AuthType.VISIBLE, null, user);
@GetMapping("/viewList/{domainId}")
public List<ViewResp> getViewList(@PathVariable("domainId") Long domainId) {
return semanticInterpreter.getViewList(domainId);
}
@PostMapping("/dimension/page")
public PageInfo<DimensionResp> getDimension(@RequestBody PageDimensionReq pageDimensionReq,
HttpServletRequest request,
HttpServletResponse response) {
public PageInfo<DimensionResp> getDimension(@RequestBody PageDimensionReq pageDimensionReq) {
return semanticInterpreter.getDimensionPage(pageDimensionReq);
}

3
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java
View File

@@ -13,7 +13,6 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
import java.util.List;
import java.util.Set;
public interface ChatService {
@@ -22,7 +21,7 @@ public interface ChatService {
* @param chatId
* @return
*/
Set<Long> getContextModel(Integer chatId);
Long getContextModel(Integer chatId);
ChatContext getOrCreateContext(int chatId);

210
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java
View File

@@ -1,41 +1,27 @@
package com.tencent.supersonic.chat.server.service;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.DataInfo;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ModelInfo;
import com.tencent.supersonic.chat.core.config.AggregatorConfig;
import com.tencent.supersonic.chat.api.pojo.response.ViewInfo;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.chat.server.service.impl.SchemaService;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.ModelCluster;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -43,16 +29,20 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
public class SemanticService {
@Autowired
private SchemaService schemaService;
@Autowired
private ConfigService configService;
@Autowired
private AggregatorConfig aggregatorConfig;
private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
@@ -60,31 +50,32 @@ public class SemanticService {
return schemaService.getSemanticSchema();
}
public ModelSchema getModelSchema(Long id) {
return schemaService.getModelSchema(id);
public ViewSchema getModelSchema(Long id) {
return schemaService.getViewSchema(id);
}
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
if (parseInfo != null && parseInfo.getModelId() > 0) {
EntityInfo entityInfo = getEntityInfo(parseInfo.getModelId());
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getModelInfo() == null) {
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, ViewSchema viewSchema, User user) {
if (parseInfo != null && parseInfo.getViewId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(viewSchema);
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getViewInfo() == null) {
entityInfo.setMetrics(null);
entityInfo.setDimensions(null);
return entityInfo;
}
String primaryKey = entityInfo.getModelInfo().getPrimaryKey();
String primaryKey = entityInfo.getViewInfo().getPrimaryKey();
if (StringUtils.isNotBlank(primaryKey)) {
String modelInfoId = "";
String entityId = "";
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
.equals(primaryKey)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
modelInfoId = chatFilter.getValue().toString();
entityId = chatFilter.getValue().toString();
}
}
}
entityInfo.setEntityId(entityId);
try {
setMainModel(entityInfo, parseInfo, modelInfoId, user);
fillEntityInfoValue(entityInfo, viewSchema, user);
return entityInfo;
} catch (Exception e) {
log.error("setMainModel error", e);
@@ -94,77 +85,50 @@ public class SemanticService {
return null;
}
public EntityInfo getEntityInfo(Long model) {
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(model);
if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) {
return new EntityInfo();
}
return getEntityInfo(chaConfigRichDesc);
}
private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) {
private EntityInfo getEntityBasicInfo(ViewSchema viewSchema) {
EntityInfo entityInfo = new EntityInfo();
Long modelId = chaConfigRichDesc.getModelId();
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(modelId)) {
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
ModelSchema modelSchema = schemaService.getModelSchema(modelId);
if (Objects.isNull(modelSchema) || Objects.isNull(modelSchema.getEntity())) {
return entityInfo;
}
ModelInfo modelInfo = new ModelInfo();
modelInfo.setItemId(modelId.intValue());
modelInfo.setName(modelSchema.getModel().getName());
modelInfo.setWords(modelSchema.getModel().getAlias());
modelInfo.setBizName(modelSchema.getModel().getBizName());
if (Objects.nonNull(modelSchema.getEntity())) {
modelInfo.setPrimaryKey(modelSchema.getEntity().getBizName());
}
entityInfo.setModelInfo(modelInfo);
List<DataInfo> dimensions = new ArrayList<>();
List<DataInfo> metrics = new ArrayList<>();
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig())
&& Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) {
ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig()
.getChatDefaultConfig();
if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())) {
for (SchemaElement dimensionDesc : chatDefaultConfig.getDimensions()) {
DataInfo mainEntityDimension = new DataInfo();
mainEntityDimension.setItemId(dimensionDesc.getId().intValue());
mainEntityDimension.setName(dimensionDesc.getName());
mainEntityDimension.setBizName(dimensionDesc.getBizName());
dimensions.add(mainEntityDimension);
}
entityInfo.setDimensions(dimensions);
}
if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
for (SchemaElement metricDesc : chatDefaultConfig.getMetrics()) {
DataInfo dataInfo = new DataInfo();
dataInfo.setName(metricDesc.getName());
dataInfo.setBizName(metricDesc.getBizName());
dataInfo.setItemId(metricDesc.getId().intValue());
metrics.add(dataInfo);
}
entityInfo.setMetrics(metrics);
}
}
if (viewSchema == null) {
return entityInfo;
}
Long viewId = viewSchema.getView().getView();
ViewInfo viewInfo = new ViewInfo();
viewInfo.setItemId(viewId.intValue());
viewInfo.setName(viewSchema.getView().getName());
viewInfo.setWords(viewSchema.getView().getAlias());
viewInfo.setBizName(viewSchema.getView().getBizName());
if (Objects.nonNull(viewSchema.getEntity())) {
viewInfo.setPrimaryKey(viewSchema.getEntity().getBizName());
}
entityInfo.setViewInfo(viewInfo);
TagTypeDefaultConfig tagTypeDefaultConfig = viewSchema.getTagTypeDefaultConfig();
if (tagTypeDefaultConfig == null) {
return entityInfo;
}
List<DataInfo> dimensions = tagTypeDefaultConfig.getDimensionIds().stream()
.map(id -> {
SchemaElement element = viewSchema.getElement(SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> metrics = tagTypeDefaultConfig.getDimensionIds().stream()
.map(id -> {
SchemaElement element = viewSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
entityInfo.setDimensions(dimensions);
entityInfo.setMetrics(metrics);
return entityInfo;
}
public void setMainModel(EntityInfo modelInfo, SemanticParseInfo parseInfo, String entity, User user) {
if (StringUtils.isEmpty(entity)) {
return;
}
List<String> entities = Collections.singletonList(entity);
SemanticQueryResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities,
user);
public void fillEntityInfoValue(EntityInfo entityInfo, ViewSchema viewSchema, User user) {
SemanticQueryResp queryResultWithColumns =
getQueryResultWithSchemaResp(entityInfo, viewSchema, user);
if (queryResultWithColumns != null) {
if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
&& queryResultWithColumns.getResultList().size() > 0) {
@@ -174,47 +138,38 @@ public class SemanticService {
if (entry.getValue() == null || entryKey == null) {
continue;
}
modelInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
modelInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo,
List<String> entities, User user) {
if (CollectionUtils.isEmpty(entities)) {
return null;
}
ModelSchema modelSchema = schemaService.getModelSchema(parseInfo.getModelId());
modelInfo.setEntityId(entities.get(0));
public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, ViewSchema viewSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(parseInfo.getModelId())));
semanticParseInfo.setView(viewSchema.getView());
semanticParseInfo.setQueryType(QueryType.TAG);
semanticParseInfo.setMetrics(getMetrics(modelInfo));
semanticParseInfo.setDimensions(getDimensions(modelInfo));
semanticParseInfo.setMetrics(getMetrics(entityInfo));
semanticParseInfo.setDimensions(getDimensions(entityInfo));
DateConf dateInfo = new DateConf();
int unit = 1;
ChatConfigResp chatConfigInfo =
configService.fetchConfigByModelId(modelSchema.getModel().getId());
if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(chatConfigInfo.getChatDetailConfig())
&& Objects.nonNull(chatConfigInfo.getChatDetailConfig().getChatDefaultConfig())) {
ChatDefaultConfigReq chatDefaultConfig = chatConfigInfo.getChatDetailConfig().getChatDefaultConfig();
unit = chatDefaultConfig.getUnit();
TimeDefaultConfig timeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig();
if (Objects.nonNull(timeDefaultConfig)) {
unit = timeDefaultConfig.getUnit();
String date = LocalDate.now().plusDays(-unit).toString();
dateInfo.setDateMode(DateMode.BETWEEN);
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(date);
dateInfo.setEndDate(date);
} else {
dateInfo.setUnit(unit);
dateInfo.setDateMode(DateMode.RECENT);
dateInfo.setDateMode(DateConf.DateMode.RECENT);
}
semanticParseInfo.setDateInfo(dateInfo);
// add filter
QueryFilter chatFilter = getQueryFilter(modelInfo, entities);
QueryFilter chatFilter = getQueryFilter(entityInfo);
Set<QueryFilter> chatFilters = new LinkedHashSet();
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
@@ -229,16 +184,11 @@ public class SemanticService {
return queryResultWithColumns;
}
private QueryFilter getQueryFilter(EntityInfo modelInfo, List<String> entities) {
private QueryFilter getQueryFilter(EntityInfo entityInfo) {
QueryFilter chatFilter = new QueryFilter();
if (entities.size() == 1) {
chatFilter.setValue(entities.get(0));
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
} else {
chatFilter.setValue(entities);
chatFilter.setOperator(FilterOperatorEnum.IN);
}
chatFilter.setBizName(getEntityPrimaryName(modelInfo));
chatFilter.setValue(entityInfo.getEntityId());
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
chatFilter.setBizName(getEntityPrimaryName(entityInfo));
return chatFilter;
}
@@ -271,8 +221,8 @@ public class SemanticService {
return metrics;
}
private String getEntityPrimaryName(EntityInfo modelInfo) {
return modelInfo.getModelInfo().getPrimaryKey();
private String getEntityPrimaryName(EntityInfo entityInfo) {
return entityInfo.getViewInfo().getPrimaryKey();
}
}

7
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java
View File

@@ -31,7 +31,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -52,7 +51,7 @@ public class ChatServiceImpl implements ChatService {
}
@Override
public Set<Long> getContextModel(Integer chatId) {
public Long getContextModel(Integer chatId) {
if (Objects.isNull(chatId)) {
return null;
}
@@ -61,8 +60,8 @@ public class ChatServiceImpl implements ChatService {
return null;
}
SemanticParseInfo originalSemanticParse = chatContext.getParseInfo();
if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getModel().getModelIds())) {
return originalSemanticParse.getModel().getModelIds();
if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getViewId())) {
return originalSemanticParse.getViewId();
}
return null;
}

34
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java
View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
@@ -173,7 +173,7 @@ public class ConfigServiceImpl implements ConfigService {
}
private ItemVisibilityInfo fetchVisibilityDescByConfig(ItemVisibility visibility,
ModelSchema modelSchema) {
ViewSchema modelSchema) {
ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo();
List<Long> dimIdAllList = chatConfigHelper.generateAllDimIdList(modelSchema);
@@ -215,22 +215,22 @@ public class ConfigServiceImpl implements ConfigService {
}
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (modelSchema == null) {
ViewSchema viewSchema = semanticService.getModelSchema(modelId);
if (viewSchema == null) {
return chatConfigRich;
}
chatConfigRich.setBizName(modelSchema.getModel().getBizName());
chatConfigRich.setModelName(modelSchema.getModel().getName());
chatConfigRich.setBizName(viewSchema.getView().getBizName());
chatConfigRich.setModelName(viewSchema.getView().getName());
chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(modelSchema, chatConfigResp));
chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(modelSchema, chatConfigRich, chatConfigResp));
chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(viewSchema, chatConfigResp));
chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(viewSchema, chatConfigRich, chatConfigResp));
return chatConfigRich;
}
private ChatDetailRichConfigResp fillChatDetailRichConfig(ModelSchema modelSchema,
ChatConfigRichResp chatConfigRich,
ChatConfigResp chatConfigResp) {
private ChatDetailRichConfigResp fillChatDetailRichConfig(ViewSchema modelSchema,
ChatConfigRichResp chatConfigRich,
ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) {
return null;
}
@@ -247,7 +247,7 @@ public class ConfigServiceImpl implements ConfigService {
return detailRichConfig;
}
private EntityRichInfoResp generateRichEntity(Entity entity, ModelSchema modelSchema) {
private EntityRichInfoResp generateRichEntity(Entity entity, ViewSchema modelSchema) {
EntityRichInfoResp entityRichInfo = new EntityRichInfoResp();
if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) {
return entityRichInfo;
@@ -260,7 +260,7 @@ public class ConfigServiceImpl implements ConfigService {
return entityRichInfo;
}
private ChatAggRichConfigResp fillChatAggRichConfig(ModelSchema modelSchema, ChatConfigResp chatConfigResp) {
private ChatAggRichConfigResp fillChatAggRichConfig(ViewSchema modelSchema, ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {
return null;
}
@@ -277,7 +277,7 @@ public class ConfigServiceImpl implements ConfigService {
}
private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig,
ModelSchema modelSchema,
ViewSchema modelSchema,
ItemVisibilityInfo itemVisibilityInfo) {
ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp();
if (Objects.isNull(chatDefaultConfig)) {
@@ -327,7 +327,7 @@ public class ConfigServiceImpl implements ConfigService {
}
private List<KnowledgeInfoReq> fillKnowledgeBizName(List<KnowledgeInfoReq> knowledgeInfos,
ModelSchema modelSchema) {
ViewSchema modelSchema) {
if (CollectionUtils.isEmpty(knowledgeInfos)) {
return new ArrayList<>();
}
@@ -347,9 +347,9 @@ public class ConfigServiceImpl implements ConfigService {
@Override
public List<ChatConfigRichResp> getAllChatRichConfig() {
List<ChatConfigRichResp> chatConfigRichInfoList = new ArrayList<>();
List<ModelSchema> modelSchemas = semanticInterpreter.getModelSchema();
List<ViewSchema> modelSchemas = semanticInterpreter.getViewSchema();
modelSchemas.stream().forEach(modelSchema -> {
ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getModel().getId());
ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getView().getId());
if (Objects.nonNull(chatConfigRichInfo)) {
chatConfigRichInfoList.add(chatConfigRichInfo);
}

62
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java
View File

@@ -1,22 +1,25 @@
package com.tencent.supersonic.chat.server.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.plugin.Plugin;
import com.tencent.supersonic.chat.core.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.core.plugin.event.PluginAddEvent;
import com.tencent.supersonic.chat.core.plugin.event.PluginDelEvent;
import com.tencent.supersonic.chat.core.plugin.event.PluginUpdateEvent;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample;
import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository;
import com.tencent.supersonic.chat.server.service.PluginService;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@@ -24,12 +27,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Slf4j
@@ -91,27 +88,27 @@ public class PluginServiceImpl implements PluginService {
@Override
public List<Plugin> query(PluginQueryReq pluginQueryReq) {
PluginDOExample pluginDOExample = new PluginDOExample();
pluginDOExample.createCriteria();
QueryWrapper<PluginDO> queryWrapper = new QueryWrapper<>();
if (StringUtils.isNotBlank(pluginQueryReq.getType())) {
pluginDOExample.getOredCriteria().get(0).andTypeEqualTo(pluginQueryReq.getType());
queryWrapper.lambda().eq(PluginDO::getType, pluginQueryReq.getType());
}
if (StringUtils.isNotBlank(pluginQueryReq.getModel())) {
pluginDOExample.getOredCriteria().get(0).andModelLike('%' + pluginQueryReq.getModel() + '%');
if (StringUtils.isNotBlank(pluginQueryReq.getView())) {
queryWrapper.lambda().like(PluginDO::getView, pluginQueryReq.getView());
}
if (StringUtils.isNotBlank(pluginQueryReq.getParseMode())) {
pluginDOExample.getOredCriteria().get(0).andParseModeEqualTo(pluginQueryReq.getParseMode());
queryWrapper.lambda().eq(PluginDO::getParseMode, pluginQueryReq.getParseMode());
}
if (StringUtils.isNotBlank(pluginQueryReq.getName())) {
pluginDOExample.getOredCriteria().get(0).andNameLike('%' + pluginQueryReq.getName() + '%');
queryWrapper.lambda().like(PluginDO::getName, pluginQueryReq.getName());
}
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
pluginDOExample.getOredCriteria().get(0).andPatternLike('%' + pluginQueryReq.getPattern() + '%');
queryWrapper.lambda().like(PluginDO::getPattern, pluginQueryReq.getPattern());
}
if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) {
pluginDOExample.getOredCriteria().get(0).andCreatedByEqualTo(pluginQueryReq.getCreatedBy());
queryWrapper.lambda().eq(PluginDO::getCreatedBy, pluginQueryReq.getCreatedBy());
}
List<PluginDO> pluginDOS = pluginRepository.query(pluginDOExample);
List<PluginDO> pluginDOS = pluginRepository.query(queryWrapper);
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
pluginDOS = pluginDOS.stream().filter(pluginDO ->
pluginDO.getPattern().contains(pluginQueryReq.getPattern())
@@ -175,29 +172,16 @@ public class PluginServiceImpl implements PluginService {
}, a -> a, (k1, k2) -> k1));
}
//todo
private List<Plugin> authCheck(List<Plugin> plugins, User user) {
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
List<Long> modelIdAuthorized = semanticInterpreter.getModelList(AuthType.ADMIN, null, user).stream()
.map(ModelResp::getId).collect(Collectors.toList());
plugins = plugins.stream().filter(plugin -> {
if (CollectionUtils.isEmpty(plugin.getModelList()) || plugin.isContainsAllModel()) {
return true;
}
for (Long modelId : plugin.getModelList()) {
if (modelIdAuthorized.contains(modelId)) {
return true;
}
}
return false;
}).collect(Collectors.toList());
return plugins;
}
public Plugin convert(PluginDO pluginDO) {
Plugin plugin = new Plugin();
BeanUtils.copyProperties(pluginDO, plugin);
if (StringUtils.isNotBlank(pluginDO.getModel())) {
plugin.setModelList(Arrays.stream(pluginDO.getModel().split(","))
if (StringUtils.isNotBlank(pluginDO.getView())) {
plugin.setViewList(Arrays.stream(pluginDO.getView().split(","))
.map(Long::parseLong).collect(Collectors.toList()));
}
return plugin;
@@ -210,7 +194,7 @@ public class PluginServiceImpl implements PluginService {
pluginDO.setCreatedBy(user.getName());
pluginDO.setUpdatedAt(new Date());
pluginDO.setUpdatedBy(user.getName());
pluginDO.setModel(StringUtils.join(plugin.getModelList(), ","));
pluginDO.setView(StringUtils.join(plugin.getViewList(), ","));
return pluginDO;
}
@@ -218,7 +202,7 @@ public class PluginServiceImpl implements PluginService {
BeanUtils.copyProperties(plugin, pluginDO);
pluginDO.setUpdatedAt(new Date());
pluginDO.setUpdatedBy(user.getName());
pluginDO.setModel(StringUtils.join(plugin.getModelList(), ","));
pluginDO.setView(StringUtils.join(plugin.getViewList(), ","));
return pluginDO;
}

34
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
@@ -65,14 +66,6 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
@@ -97,6 +90,15 @@ import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Component("chatQueryService")
@Primary
@@ -290,7 +292,7 @@ public class QueryServiceImpl implements QueryService {
similarQueryManager.saveSimilarQuery(SimilarQueryReq.builder().parseId(queryReq.getParseId())
.queryId(queryReq.getQueryId())
.agentId(chatQueryDO.getAgentId())
.modelId(parseInfo.getModelClusterKey())
.viewId(parseInfo.getViewId())
.queryText(queryReq.getQueryText()).build());
}
@@ -346,8 +348,9 @@ public class QueryServiceImpl implements QueryService {
}
QueryResult queryResult = semanticQuery.execute(user);
queryResult.setChatContext(semanticQuery.getParseInfo());
ViewSchema viewSchema = semanticSchema.getViewSchemaMap().get(parseInfo.getViewId());
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, user);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, viewSchema, user);
queryResult.setEntityInfo(entityInfo);
return queryResult;
}
@@ -415,7 +418,8 @@ public class QueryServiceImpl implements QueryService {
ChatParseDO chatParseDO = chatService.getParseInfo(queryId, parseId);
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
return semanticService.getEntityInfo(parseInfo, user);
ViewSchema viewSchema = schemaService.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId());
return semanticService.getEntityInfo(parseInfo, viewSchema, user);
}
private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo,
@@ -632,10 +636,10 @@ public class QueryServiceImpl implements QueryService {
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SchemaElement schemaElement = semanticSchema.getDimensions(dimensionValueReq.getElementID());
SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
Set<Long> detectModelIds = new HashSet<>();
detectModelIds.add(schemaElement.getModel());
dimensionValueReq.setModelId(schemaElement.getModel());
detectModelIds.add(schemaElement.getView());
dimensionValueReq.setModelId(schemaElement.getView());
List<String> dimensionValues = getDimensionValues(dimensionValueReq, detectModelIds);
// if the search results is null,search dimensionValue from database
if (CollectionUtils.isEmpty(dimensionValues)) {
@@ -692,7 +696,7 @@ public class QueryServiceImpl implements QueryService {
dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
queryStructReq.addModelId(dimensionValueReq.getModelId());
queryStructReq.setViewId(dimensionValueReq.getModelId());
queryStructReq.setQueryType(QueryType.ID);
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());

17
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java
View File

@@ -2,9 +2,9 @@ package com.tencent.supersonic.chat.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
@@ -13,6 +13,11 @@ import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import com.tencent.supersonic.chat.server.service.ConfigService;
import com.tencent.supersonic.chat.server.service.RecommendService;
import com.tencent.supersonic.chat.server.service.SemanticService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -20,10 +25,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
/***
* Recommend Service impl
@@ -47,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService {
if (Objects.isNull(modelId)) {
return new RecommendResp();
}
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
ViewSchema modelSchema = semanticService.getModelSchema(modelId);
if (Objects.isNull(modelSchema)) {
return new RecommendResp();
}
@@ -79,7 +80,7 @@ public class RecommendServiceImpl implements RecommendService {
.limit(limit)
.map(dimSchemaDesc -> {
SchemaElement item = new SchemaElement();
item.setModel(modelId);
item.setView(modelId);
item.setName(dimSchemaDesc.getName());
item.setBizName(dimSchemaDesc.getBizName());
item.setId(dimSchemaDesc.getId());
@@ -93,7 +94,7 @@ public class RecommendServiceImpl implements RecommendService {
.limit(limit)
.map(metricSchemaDesc -> {
SchemaElement item = new SchemaElement();
item.setModel(modelId);
item.setView(modelId);
item.setName(metricSchemaDesc.getName());
item.setBizName(metricSchemaDesc.getBizName());
item.setId(metricSchemaDesc.getId());

13
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java
View File

@@ -3,14 +3,15 @@ package com.tencent.supersonic.chat.server.service.impl;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.concurrent.TimeUnit;
@Service
@Slf4j
public class SchemaService {
@@ -27,13 +28,13 @@ public class SchemaService {
@Override
public SemanticSchema load(String key) {
log.info("load getDomainSchemaInfo cache [{}]", key);
return new SemanticSchema(semanticInterpreter.getModelSchema());
return new SemanticSchema(semanticInterpreter.getViewSchema());
}
}
);
public ModelSchema getModelSchema(Long id) {
return semanticInterpreter.getModelSchema(id, true);
public ViewSchema getViewSchema(Long id) {
return semanticInterpreter.getViewSchema(id, true);
}
public SemanticSchema getSemanticSchema() {

47
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.server.service.impl;
import com.github.benmanes.caffeine.cache.Cache;
import com.google.common.collect.Lists;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
@@ -15,11 +14,12 @@ import com.tencent.supersonic.chat.api.pojo.response.SearchResult;
import com.tencent.supersonic.chat.core.agent.Agent;
import com.tencent.supersonic.chat.core.knowledge.DictWord;
import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.chat.core.knowledge.ModelInfoStat;
import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat;
import com.tencent.supersonic.chat.core.mapper.MapperHelper;
import com.tencent.supersonic.chat.core.mapper.MatchText;
import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType;
import com.tencent.supersonic.chat.core.mapper.SearchMatchStrategy;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
import com.tencent.supersonic.chat.core.utils.NatureHelper;
import com.tencent.supersonic.chat.server.service.AgentService;
@@ -28,6 +28,13 @@ import com.tencent.supersonic.chat.server.service.ConfigService;
import com.tencent.supersonic.chat.server.service.SearchService;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
@@ -40,12 +47,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
/**
@@ -86,13 +87,13 @@ public class SearchServiceImpl implements SearchService {
// 2.get meta info
SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema();
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
final Map<Long, String> modelToName = semanticSchemaDb.getModelIdToName();
final Map<Long, String> modelToName = semanticSchemaDb.getViewIdToName();
// 3.detect by segment
List<Term> originals = HanlpHelper.getTerms(queryText);
log.info("hanlp parse result: {}", originals);
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
Set<Long> detectModelIds = mapperHelper.getModelIds(queryReq.getModelId(), agentService.getAgent(agentId));
Set<Long> detectModelIds = mapperHelper.getViewIds(queryReq.getModelId(), agentService.getAgent(agentId));
QueryContext queryContext = new QueryContext();
BeanUtils.copyProperties(queryReq, queryContext);
@@ -116,7 +117,7 @@ public class SearchServiceImpl implements SearchService {
log.info("searchTextEntry:{},queryReq:{}", searchTextEntry, queryReq);
Set<SearchResult> searchResults = new LinkedHashSet();
ModelInfoStat modelStat = NatureHelper.getModelStat(originals);
ViewInfoStat modelStat = NatureHelper.getViewStat(originals);
List<Long> possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getModelId());
@@ -132,7 +133,7 @@ public class SearchServiceImpl implements SearchService {
for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
Set<SearchResult> searchResultSet = searchDimensionValue(metricsDb, modelToName,
modelStat.getMetricModelCount(), existMetricAndDimension,
modelStat.getMetricViewCount(), existMetricAndDimension,
matchText, natureToNameMap, natureToNameEntry, queryReq.getQueryFilters());
searchResults.addAll(searchResultSet);
@@ -141,7 +142,7 @@ public class SearchServiceImpl implements SearchService {
}
private List<Long> getPossibleModels(QueryReq queryCtx, List<Term> originals,
ModelInfoStat modelStat, Long webModelId) {
ViewInfoStat modelStat, Long webModelId) {
if (Objects.nonNull(webModelId) && webModelId > 0) {
List<Long> result = new ArrayList<>();
@@ -149,22 +150,22 @@ public class SearchServiceImpl implements SearchService {
return result;
}
List<Long> possibleModels = NatureHelper.selectPossibleModels(originals);
List<Long> possibleModels = NatureHelper.selectPossibleViews(originals);
Set<Long> contextModel = chatService.getContextModel(queryCtx.getChatId());
Long contextModel = chatService.getContextModel(queryCtx.getChatId());
log.debug("possibleModels:{},modelStat:{},contextModel:{}", possibleModels, modelStat, contextModel);
// If nothing is recognized or only metric are present, then add the contextModel.
if (nothingOrOnlyMetric(modelStat)) {
return contextModel.stream().filter(modelId -> modelId > 0).collect(Collectors.toList());
return Lists.newArrayList(contextModel);
}
return possibleModels;
}
private boolean nothingOrOnlyMetric(ModelInfoStat modelStat) {
return modelStat.getMetricModelCount() >= 0 && modelStat.getDimensionModelCount() <= 0
&& modelStat.getDimensionValueModelCount() <= 0 && modelStat.getModelCount() <= 0;
private boolean nothingOrOnlyMetric(ViewInfoStat modelStat) {
return modelStat.getMetricViewCount() >= 0 && modelStat.getDimensionViewCount() <= 0
&& modelStat.getDimensionValueViewCount() <= 0 && modelStat.getViewCount() <= 0;
}
private boolean effectiveModel(Long contextModel) {
@@ -184,7 +185,7 @@ public class SearchServiceImpl implements SearchService {
String nature = natureToNameEntry.getKey();
String wordName = natureToNameEntry.getValue();
Long modelId = NatureHelper.getModelId(nature);
Long modelId = NatureHelper.getViewId(nature);
SchemaElementType schemaElementType = NatureHelper.convertToElementType(nature);
if (SchemaElementType.ENTITY.equals(schemaElementType)) {
@@ -261,7 +262,7 @@ public class SearchServiceImpl implements SearchService {
return Lists.newArrayList();
}
return metricsDb.stream()
.filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getModel()))
.filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getView()))
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.flatMap(entry -> {
List<String> result = new ArrayList<>();
@@ -285,7 +286,7 @@ public class SearchServiceImpl implements SearchService {
if (CollectionUtils.isEmpty(possibleModels)) {
return true;
}
Long model = NatureHelper.getModelId(nature);
Long model = NatureHelper.getViewId(nature);
return possibleModels.contains(model);
})
.map(nature -> {
@@ -308,7 +309,7 @@ public class SearchServiceImpl implements SearchService {
for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
List<ModelWithSemanticType> dimensionMetricClassIds = hanlpMapResult.getNatures().stream()
.map(nature -> new ModelWithSemanticType(NatureHelper.getModelId(nature),
.map(nature -> new ModelWithSemanticType(NatureHelper.getViewId(nature),
NatureHelper.convertToElementType(nature)))
.filter(entry -> matchCondition(entry, possibleModels)).collect(Collectors.toList());

11
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java
View File

@@ -1,17 +1,18 @@
package com.tencent.supersonic.chat.server.service.impl;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.core.knowledge.DictWord;
import com.tencent.supersonic.chat.core.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
@@ -21,13 +22,13 @@ public class WordService {
public List<DictWord> getAllDictWords() {
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getModelSchema());
SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getViewSchema());
List<DictWord> words = new ArrayList<>();
addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words);
addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words);
addWordsByType(DictWordType.MODEL, semanticSchema.getModels(), words);
addWordsByType(DictWordType.VIEW, semanticSchema.getViews(), words);
addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words);
addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words);

6
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.util;
import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
@@ -56,7 +56,7 @@ public class ChatConfigHelper {
return chatConfig;
}
public List<Long> generateAllDimIdList(ModelSchema modelSchema) {
public List<Long> generateAllDimIdList(ViewSchema modelSchema) {
if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getDimensions())) {
return new ArrayList<>();
}
@@ -65,7 +65,7 @@ public class ChatConfigHelper {
return new ArrayList<>(dimIdAndDescPair.keySet());
}
public List<Long> generateAllMetricIdList(ModelSchema modelSchema) {
public List<Long> generateAllMetricIdList(ViewSchema modelSchema) {
if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getMetrics())) {
return new ArrayList<>();
}

37
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java
View File

@@ -1,11 +1,8 @@
package com.tencent.supersonic.chat.server.util;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
@@ -19,6 +16,13 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO;
import com.tencent.supersonic.chat.server.service.ConfigService;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -28,12 +32,9 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
@Component
public class DictMetaHelper {
@@ -56,13 +57,13 @@ public class DictMetaHelper {
dimValueDOList = generateDimValueInfoByModel(modelIds);
break;
case OFFLINE_FULL:
List<ModelSchema> modelSchemaDescList = semanticInterpreter.getModelSchema();
List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema();
if (CollectionUtils.isEmpty(modelSchemaDescList)) {
break;
}
Map<Long, ModelSchema> modelIdAndDescPair = modelSchemaDescList.stream()
.collect(Collectors.toMap(a -> a.getModel().getId(), schema -> schema, (k1, k2) -> k1));
Map<Long, ViewSchema> modelIdAndDescPair = modelSchemaDescList.stream()
.collect(Collectors.toMap(a -> a.getView().getId(), schema -> schema, (k1, k2) -> k1));
if (!CollectionUtils.isEmpty(modelIdAndDescPair)) {
modelIds.addAll(modelIdAndDescPair.keySet());
dimValueDOList = generateDimValueInfoByModel(modelIds);
@@ -87,12 +88,12 @@ public class DictMetaHelper {
return dimValueDOList;
}
List<ModelSchema> modelSchemaDescList = semanticInterpreter.getModelSchema();
List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema();
if (CollectionUtils.isEmpty(modelSchemaDescList)) {
return dimValueDOList;
}
Map<Long, ModelSchema> modelIdAndDescPair = modelSchemaDescList.stream()
.collect(Collectors.toMap(a -> a.getModel().getId(), a -> a, (k1, k2) -> k1));
Map<Long, ViewSchema> modelIdAndDescPair = modelSchemaDescList.stream()
.collect(Collectors.toMap(a -> a.getView().getId(), a -> a, (k1, k2) -> k1));
for (Long modelId : modelAndDimMap.keySet()) {
if (!modelIdAndDescPair.containsKey(modelId)) {
@@ -116,7 +117,7 @@ public class DictMetaHelper {
private List<DimValueDO> generateDimValueInfoByModel(Set<Long> modelIds) {
List<DimValueDO> dimValueDOList = new ArrayList<>();
List<ModelSchema> modelSchemaDescList = semanticInterpreter.getModelSchema(new ArrayList<>(modelIds));
List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema(new ArrayList<>(modelIds));
if (CollectionUtils.isEmpty(modelSchemaDescList)) {
return dimValueDOList;
}
@@ -124,7 +125,7 @@ public class DictMetaHelper {
modelSchemaDescList.forEach(modelSchemaDesc -> {
Map<Long, SchemaElement> dimIdAndDescPair = modelSchemaDesc.getDimensions().stream()
.collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1));
fillDimValueDOList(dimValueDOList, modelSchemaDesc.getModel().getId(), dimIdAndDescPair);
fillDimValueDOList(dimValueDOList, modelSchemaDesc.getView().getId(), dimIdAndDescPair);
});

276
chat/server/src/main/resources/mapper/PluginDOMapper.xml
View File

@@ -1,276 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.chat.server.persistence.mapper.PluginDOMapper">
<resultMap id="BaseResultMap" type="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
<id column="id" jdbcType="BIGINT" property="id" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="model" jdbcType="VARCHAR" property="model" />
<result column="pattern" jdbcType="VARCHAR" property="pattern" />
<result column="parse_mode" jdbcType="VARCHAR" property="parseMode" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
<result column="parse_mode_config" jdbcType="LONGVARCHAR" property="parseModeConfig" />
<result column="config" jdbcType="LONGVARCHAR" property="config" />
<result column="comment" jdbcType="LONGVARCHAR" property="comment" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, type, model, pattern, parse_mode, name, created_at, created_by, updated_at,
updated_by
</sql>
<sql id="Blob_Column_List">
parse_mode_config, config, comment
</sql>
<select id="selectByExampleWithBLOBs" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="limitStart != null and limitStart>=0">
limit #{limitStart} , #{limitEnd}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.Long" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_plugin
where id = #{id,jdbcType=BIGINT}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
delete from s2_plugin
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
insert into s2_plugin (id, type, model,
pattern, parse_mode, name,
created_at, created_by, updated_at,
updated_by, parse_mode_config, config,
comment)
values (#{id,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR}, #{model,jdbcType=VARCHAR},
#{pattern,jdbcType=VARCHAR}, #{parseMode,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{parseModeConfig,jdbcType=LONGVARCHAR}, #{config,jdbcType=LONGVARCHAR},
#{comment,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
insert into s2_plugin
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="type != null">
type,
</if>
<if test="model != null">
model,
</if>
<if test="pattern != null">
pattern,
</if>
<if test="parseMode != null">
parse_mode,
</if>
<if test="name != null">
name,
</if>
<if test="createdAt != null">
created_at,
</if>
<if test="createdBy != null">
created_by,
</if>
<if test="updatedAt != null">
updated_at,
</if>
<if test="updatedBy != null">
updated_by,
</if>
<if test="parseModeConfig != null">
parse_mode_config,
</if>
<if test="config != null">
config,
</if>
<if test="comment != null">
comment,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=BIGINT},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="model != null">
#{model,jdbcType=VARCHAR},
</if>
<if test="pattern != null">
#{pattern,jdbcType=VARCHAR},
</if>
<if test="parseMode != null">
#{parseMode,jdbcType=VARCHAR},
</if>
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
#{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
#{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
#{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
#{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
#{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
#{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
#{comment,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample" resultType="java.lang.Long">
select count(*) from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByPrimaryKeySelective" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
update s2_plugin
<set>
<if test="type != null">
type = #{type,jdbcType=VARCHAR},
</if>
<if test="model != null">
model = #{model,jdbcType=VARCHAR},
</if>
<if test="pattern != null">
pattern = #{pattern,jdbcType=VARCHAR},
</if>
<if test="parseMode != null">
parse_mode = #{parseMode,jdbcType=VARCHAR},
</if>
<if test="name != null">
name = #{name,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
created_at = #{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
created_by = #{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
updated_by = #{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
config = #{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
comment = #{comment,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
update s2_plugin
set type = #{type,jdbcType=VARCHAR},
model = #{model,jdbcType=VARCHAR},
pattern = #{pattern,jdbcType=VARCHAR},
parse_mode = #{parseMode,jdbcType=VARCHAR},
name = #{name,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
config = #{config,jdbcType=LONGVARCHAR},
comment = #{comment,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO">
update s2_plugin
set type = #{type,jdbcType=VARCHAR},
model = #{model,jdbcType=VARCHAR},
pattern = #{pattern,jdbcType=VARCHAR},
parse_mode = #{parseMode,jdbcType=VARCHAR},
name = #{name,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
</mapper>

6
chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.server.processor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
@@ -115,7 +115,7 @@ class MetricCheckProcessorTest {
* 访问用户数 drill down dimension is 部门, and 部门 is necessary, 部门 need in select and group by or where expressions
*/
private SemanticSchema mockModelSchema() {
ModelSchema modelSchema = new ModelSchema();
ViewSchema modelSchema = new ViewSchema();
Set<SchemaElement> metrics = Sets.newHashSet(
mockElement(1L, "访问次数", SchemaElementType.METRIC,
Lists.newArrayList(RelatedSchemaElement.builder().dimensionId(2L).isNecessary(false).build(),
@@ -129,7 +129,7 @@ class MetricCheckProcessorTest {
}
private SemanticSchema mockModelSchemaNoDimensionSetting() {
ModelSchema modelSchema = new ModelSchema();
ViewSchema modelSchema = new ViewSchema();
Set<SchemaElement> metrics = Sets.newHashSet(
mockElement(1L, "访问次数", SchemaElementType.METRIC, Lists.newArrayList()),
mockElement(2L, "访问用户数", SchemaElementType.METRIC, Lists.newArrayList())

21
chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java
View File

@@ -5,21 +5,22 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* QueryReqBuilderTest
*/
@@ -29,9 +30,9 @@ class QueryReqBuilderTest {
void buildS2SQLReq() {
init();
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.addModelId(1L);
queryStructReq.setViewId(1L);
queryStructReq.setViewName("内容库");
queryStructReq.setQueryType(QueryType.METRIC);
queryStructReq.setModelName("内容库");
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.UNKNOWN);
@@ -51,18 +52,18 @@ class QueryReqBuilderTest {
orders.add(order);
queryStructReq.setOrders(orders);
QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq);
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, SUM(pv) AS pv FROM 内容库 "
+ "WHERE (sys_imp_date IN ('2023-08-01')) GROUP "
+ "BY department ORDER BY uv LIMIT 2000", querySqlReq.getSql());
+ "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql());
queryStructReq.setQueryType(QueryType.TAG);
querySqlReq = queryStructReq.convert(queryStructReq);
querySQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "ORDER BY uv LIMIT 2000",
querySqlReq.getSql());
querySQLReq.getSql());
}