topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 04:57:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/showcase' into feature/lxw

Browse Source 
# Conflicts:
#	chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
#	chat/core/src/main/resources/mapper/ChatQueryDOMapper.xml
This commit is contained in:
jolunoluo
2023-09-25 16:59:31 +08:00
parent 51f62438cf 65653c0ee2
commit e1772c25c4
106 changed files with 2495 additions and 1162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -17,3 +17,4 @@ assembly/runtime/*
**/.flattened-pom.xml
chm_db/
__pycache__/
/dict

2
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/service/AuthService.java
View File

@@ -10,7 +10,7 @@ public interface AuthService {
List<AuthGroup> queryAuthGroups(String domainId, Integer groupId);
void updateAuthGroup(AuthGroup group);
void addOrUpdateAuthGroup(AuthGroup group);
void removeAuthGroup(AuthGroup group);

2
auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java
View File

@@ -53,7 +53,7 @@ public class AuthServiceImpl implements AuthService {
}
@Override
public void updateAuthGroup(AuthGroup group) {
public void addOrUpdateAuthGroup(AuthGroup group) {
Gson g = new Gson();
if (group.getGroupId() == null) {
int nextGroupId = 1;

4
auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/rest/AuthController.java
View File

@@ -40,7 +40,7 @@ public class AuthController {
@PostMapping("/createGroup")
public void newAuthGroup(@RequestBody AuthGroup group) {
group.setGroupId(null);
authService.updateAuthGroup(group);
authService.addOrUpdateAuthGroup(group);
}
@PostMapping("/removeGroup")
@@ -58,7 +58,7 @@ public class AuthController {
if (group.getGroupId() == null || group.getGroupId() == 0) {
throw new RuntimeException("groupId is empty");
}
authService.updateAuthGroup(group);
authService.addOrUpdateAuthGroup(group);
}
/**

17
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java
View File

@@ -7,6 +7,7 @@ import java.util.Map;
import java.util.stream.Collectors;
public class SemanticSchema implements Serializable {
private List<ModelSchema> modelSchemaList;
public SemanticSchema(List<ModelSchema> modelSchemaList) {
@@ -34,12 +35,28 @@ public class SemanticSchema implements Serializable {
return dimensions;
}
public List<SchemaElement> getDimensions(Long modelId) {
List<SchemaElement> dimensions = getDimensions();
return getElementsByModelId(modelId, dimensions);
}
public List<SchemaElement> getMetrics() {
List<SchemaElement> metrics = new ArrayList<>();
modelSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics()));
return metrics;
}
public List<SchemaElement> getMetrics(Long modelId) {
List<SchemaElement> metrics = getMetrics();
return getElementsByModelId(modelId, metrics);
}
private List<SchemaElement> getElementsByModelId(Long modelId, List<SchemaElement> elements) {
return elements.stream()
.filter(schemaElement -> modelId.equals(schemaElement.getModel()))
.collect(Collectors.toList());
}
public List<SchemaElement> getModels() {
List<SchemaElement> models = new ArrayList<>();
modelSchemaList.stream().forEach(d -> models.add(d.getModel()));

19
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java
View File

@@ -1,25 +1,20 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.HashSet;
import java.util.Set;
import lombok.Data;
@Data
public class QueryDataReq {
String queryMode;
SchemaElement model;
Set<SchemaElement> metrics = new HashSet<>();
Set<SchemaElement> dimensions = new HashSet<>();
Set<QueryFilter> dimensionFilters = new HashSet<>();
Set<QueryFilter> metricFilters = new HashSet<>();
private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
private Set<Order> orders = new HashSet<>();
private User user;
private Set<SchemaElement> metrics = new HashSet<>();
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<QueryFilter> dimensionFilters = new HashSet<>();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;
private Long queryId = 7L;
private Integer parseId = 2;
}

27
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/DateFieldCorrector.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DateFieldCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
String sql = semanticCorrectInfo.getSql();
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sql);
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DATE_FIELD)) {
String currentDate = DSLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId());
sql = SqlParserUpdateHelper.addWhere(sql, DATE_FIELD, currentDate);
}
semanticCorrectInfo.setPreSql(semanticCorrectInfo.getSql());
semanticCorrectInfo.setSql(sql);
}
}

18
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldCorrector.java
View File

@@ -1,18 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FieldCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
String preSql = semanticCorrectInfo.getSql();
semanticCorrectInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceFields(preSql,
getFieldToBizName(semanticCorrectInfo.getParseInfo().getModelId()));
semanticCorrectInfo.setSql(sql);
}
}

16
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionAliasCorrector.java
View File

@@ -1,16 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FunctionAliasCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
String replaceAlias = SqlParserUpdateHelper.replaceAlias(semanticCorrectInfo.getSql());
semanticCorrectInfo.setSql(replaceAlias);
}
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionCorrector.java
View File

@@ -1,17 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FunctionCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
String preSql = semanticCorrectInfo.getSql();
semanticCorrectInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceFunction(preSql);
semanticCorrectInfo.setSql(sql);
}
}

31
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldNameCorrector.java → chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalCorrector.java
View File

@@ -16,11 +16,39 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j
public class FieldNameCorrector extends BaseSemanticCorrector {
public class GlobalCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
replaceAlias(semanticCorrectInfo);
updateFieldNameByLinkingValue(semanticCorrectInfo);
updateFieldNameByBizName(semanticCorrectInfo);
addAggregateToMetric(semanticCorrectInfo);
}
private void addAggregateToMetric(SemanticCorrectInfo semanticCorrectInfo) {
}
private void replaceAlias(SemanticCorrectInfo semanticCorrectInfo) {
String replaceAlias = SqlParserUpdateHelper.replaceAlias(semanticCorrectInfo.getSql());
semanticCorrectInfo.setSql(replaceAlias);
}
private void updateFieldNameByBizName(SemanticCorrectInfo semanticCorrectInfo) {
Map<String, String> fieldToBizName = getFieldToBizName(semanticCorrectInfo.getParseInfo().getModelId());
String sql = SqlParserUpdateHelper.replaceFields(semanticCorrectInfo.getSql(), fieldToBizName);
semanticCorrectInfo.setSql(sql);
}
private void updateFieldNameByLinkingValue(SemanticCorrectInfo semanticCorrectInfo) {
Object context = semanticCorrectInfo.getParseInfo().getProperties().get(Constants.CONTEXT);
if (Objects.isNull(context)) {
return;
@@ -45,5 +73,4 @@ public class FieldNameCorrector extends BaseSemanticCorrector {
String sql = SqlParserUpdateHelper.replaceFieldNameByValue(preSql, fieldValueToFieldNames);
semanticCorrectInfo.setSql(sql);
}
}

15
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java Normal file
View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class GroupByCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
}
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HavingCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
}
}

48
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/QueryFilterAppend.java
View File

@@ -1,48 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class QueryFilterAppend extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException {
String queryFilter = getQueryFilter(semanticCorrectInfo.getQueryFilters());
String preSql = semanticCorrectInfo.getSql();
if (StringUtils.isNotEmpty(queryFilter)) {
log.info("add queryFilter to preSql :{}", queryFilter);
Expression expression = CCJSqlParserUtil.parseCondExpression(queryFilter);
String sql = SqlParserUpdateHelper.addWhere(preSql, expression);
semanticCorrectInfo.setPreSql(preSql);
semanticCorrectInfo.setSql(sql);
}
}
private String getQueryFilter(QueryFilters queryFilters) {
if (Objects.isNull(queryFilters) || CollectionUtils.isEmpty(queryFilters.getFilters())) {
return null;
}
return queryFilters.getFilters().stream()
.map(filter -> {
String bizNameWrap = StringUtil.getSpaceWrap(filter.getBizName());
String operatorWrap = StringUtil.getSpaceWrap(filter.getOperator().getValue());
String valueWrap = StringUtil.getCommaWrap(filter.getValue().toString());
return bizNameWrap + operatorWrap + valueWrap;
})
.collect(Collectors.joining(Constants.AND_UPPER));
}
}

3
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrector.java → chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java
View File

@@ -13,11 +13,12 @@ import net.sf.jsqlparser.expression.Expression;
import org.springframework.util.CollectionUtils;
@Slf4j
public class SelectFieldAppendCorrector extends BaseSemanticCorrector {
public class SelectCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
String preSql = semanticCorrectInfo.getSql();
if (SqlParserSelectHelper.hasAggregateFunction(preSql)) {
Expression havingExpression = SqlParserSelectHelper.getHavingExpression(preSql);
if (Objects.nonNull(havingExpression)) {

2
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableNameCorrector.java → chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableCorrector.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TableNameCorrector extends BaseSemanticCorrector {
public class TableCorrector extends BaseSemanticCorrector {
public static final String TABLE_PREFIX = "t_";

73
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldValueCorrector.java → chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java
View File

@@ -1,26 +1,92 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.util.CollectionUtils;
@Slf4j
public class FieldValueCorrector extends BaseSemanticCorrector {
public class WhereCorrector extends BaseSemanticCorrector {
@Override
public void correct(SemanticCorrectInfo semanticCorrectInfo) {
public void correct(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException {
addDateIfNotExist(semanticCorrectInfo);
parserDateDiffFunction(semanticCorrectInfo);
addQueryFilter(semanticCorrectInfo);
updateFieldValueByTechName(semanticCorrectInfo);
}
private void addQueryFilter(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException {
String queryFilter = getQueryFilter(semanticCorrectInfo.getQueryFilters());
String preSql = semanticCorrectInfo.getSql();
if (StringUtils.isNotEmpty(queryFilter)) {
log.info("add queryFilter to preSql :{}", queryFilter);
Expression expression = CCJSqlParserUtil.parseCondExpression(queryFilter);
String sql = SqlParserUpdateHelper.addWhere(preSql, expression);
semanticCorrectInfo.setPreSql(preSql);
semanticCorrectInfo.setSql(sql);
}
}
private void parserDateDiffFunction(SemanticCorrectInfo semanticCorrectInfo) {
String preSql = semanticCorrectInfo.getSql();
semanticCorrectInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceFunction(preSql);
semanticCorrectInfo.setSql(sql);
}
private void addDateIfNotExist(SemanticCorrectInfo semanticCorrectInfo) {
String sql = semanticCorrectInfo.getSql();
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sql);
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(TimeDimensionEnum.DAY.getName())) {
String currentDate = DSLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId());
sql = SqlParserUpdateHelper.addWhere(sql, TimeDimensionEnum.DAY.getName(), currentDate);
}
semanticCorrectInfo.setSql(sql);
}
private String getQueryFilter(QueryFilters queryFilters) {
if (Objects.isNull(queryFilters) || CollectionUtils.isEmpty(queryFilters.getFilters())) {
return null;
}
return queryFilters.getFilters().stream()
.map(filter -> {
String bizNameWrap = StringUtil.getSpaceWrap(filter.getBizName());
String operatorWrap = StringUtil.getSpaceWrap(filter.getOperator().getValue());
String valueWrap = StringUtil.getCommaWrap(filter.getValue().toString());
return bizNameWrap + operatorWrap + valueWrap;
})
.collect(Collectors.joining(Constants.AND_UPPER));
}
private void updateFieldValueByTechName(SemanticCorrectInfo semanticCorrectInfo) {
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Long modelId = semanticCorrectInfo.getParseInfo().getModel().getId();
List<SchemaElement> dimensions = semanticSchema.getDimensions().stream()
@@ -39,7 +105,6 @@ public class FieldValueCorrector extends BaseSemanticCorrector {
return;
}
private Map<String, Map<String, String>> getAliasAndBizNameToTechName(List<SchemaElement> dimensions) {
if (CollectionUtils.isEmpty(dimensions)) {
return new HashMap<>();

53
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java
View File

@@ -408,27 +408,20 @@ public class LLMDslParser implements SemanticParser {
protected List<String> getFieldNameList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema,
LLMParserConfig llmParserConfig) {
Set<String> results = getTopNFieldNames(modelId, semanticSchema, llmParserConfig);
Set<String> fieldNameList = getMatchedFieldNames(queryCtx, modelId, semanticSchema);
results.addAll(fieldNameList);
return new ArrayList<>(results);
}
protected Set<String> getMatchedFieldNames(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema) {
Map<Long, String> itemIdToName = getItemIdToName(modelId, semanticSchema);
Set<String> results = semanticSchema.getDimensions().stream()
.filter(schemaElement -> modelId.equals(schemaElement.getModel()))
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(llmParserConfig.getDimensionTopN())
.map(entry -> entry.getName())
.collect(Collectors.toSet());
Set<String> metrics = semanticSchema.getMetrics().stream()
.filter(schemaElement -> modelId.equals(schemaElement.getModel()))
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(llmParserConfig.getMetricTopN())
.map(entry -> entry.getName())
.collect(Collectors.toSet());
results.addAll(metrics);
List<SchemaElementMatch> matchedElements = queryCtx.getMapInfo().getMatchedElements(modelId);
if (CollectionUtils.isEmpty(matchedElements)) {
return new ArrayList<>(results);
return new HashSet<>();
}
Set<String> fieldNameList = matchedElements.stream()
.filter(schemaElementMatch -> {
@@ -447,13 +440,29 @@ public class LLMDslParser implements SemanticParser {
})
.filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
.collect(Collectors.toSet());
results.addAll(fieldNameList);
return new ArrayList<>(results);
return fieldNameList;
}
private Set<String> getTopNFieldNames(Long modelId, SemanticSchema semanticSchema,
LLMParserConfig llmParserConfig) {
Set<String> results = semanticSchema.getDimensions(modelId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(llmParserConfig.getDimensionTopN())
.map(entry -> entry.getName())
.collect(Collectors.toSet());
Set<String> metrics = semanticSchema.getMetrics(modelId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(llmParserConfig.getMetricTopN())
.map(entry -> entry.getName())
.collect(Collectors.toSet());
results.addAll(metrics);
return results;
}
protected Map<Long, String> getItemIdToName(Long modelId, SemanticSchema semanticSchema) {
return semanticSchema.getDimensions().stream()
.filter(entry -> modelId.equals(entry.getModel()))
return semanticSchema.getDimensions(modelId).stream()
.collect(Collectors.toMap(SchemaElement::getId, SchemaElement::getName, (value1, value2) -> value2));
}

1
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java
View File

@@ -72,6 +72,7 @@ public class ChatQueryController {
public Object queryData(@RequestBody QueryDataReq queryData,
HttpServletRequest request, HttpServletResponse response)
throws Exception {
queryData.setUser(UserHolder.findUser(request, response));
return queryService.executeDirectQuery(queryData, UserHolder.findUser(request, response));
}

110
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -3,11 +3,14 @@ package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
@@ -15,13 +18,15 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp;
import com.tencent.supersonic.chat.queryresponder.QueryResponder;
import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.service.QueryService;
@@ -29,25 +34,29 @@ import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.chat.service.StatisticsService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import java.util.Map;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import java.util.List;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Comparator;
import java.util.Objects;
import java.util.stream.Collectors;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.springframework.beans.BeanUtils;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Primary;
@@ -175,34 +184,26 @@ public class QueryServiceImpl implements QueryService {
ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
chatCtx.setAgentId(queryReq.getAgentId());
Long startTime = System.currentTimeMillis();
QueryResult queryResult = null;
try {
queryResult = semanticQuery.execute(queryReq.getUser());
} catch (Exception e) {
log.error("query execute failed, queryText:{}", queryReq.getQueryText(), e);
queryResult = new QueryResult();
queryResult.setQueryState(QueryState.INVALID);
QueryResult queryResult = semanticQuery.execute(queryReq.getUser());
if (queryResult != null) {
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build());
saveInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(),
queryReq.getUser().getName(), queryReq.getChatId().longValue());
queryResult.setChatContext(parseInfo);
// update chat context after a successful semantic query
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
chatCtx.setParseInfo(parseInfo);
chatService.updateContext(chatCtx);
}
chatCtx.setQueryText(queryReq.getQueryText());
chatCtx.setUser(queryReq.getUser().getName());
chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);
} else {
chatService.deleteChatQuery(queryReq.getQueryId());
}
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build());
saveInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(),
queryReq.getUser().getName(), queryReq.getChatId().longValue());
queryResult.setChatContext(parseInfo);
// update chat context after a successful semantic query
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
chatCtx.setParseInfo(parseInfo);
chatService.updateContext(chatCtx);
queryResponder.saveSolvedQuery(queryReq.getQueryText(), queryReq.getQueryId(), queryReq.getParseId());
}
chatCtx.setQueryText(queryReq.getQueryText());
chatCtx.setUser(queryReq.getUser().getName());
chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);
if (!QueryState.SUCCESS.equals(queryResult.getQueryState())) {
List<SolvedQueryRecallResp> solvedQueryRecallResps =
queryResponder.recallSolvedQuery(queryReq.getQueryText());
queryResult.setSimilarSolvedQuery(solvedQueryRecallResps);
}
return queryResult;
}
@@ -273,8 +274,52 @@ public class QueryServiceImpl implements QueryService {
@Override
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException {
SemanticQuery semanticQuery = QueryManager.createRuleQuery(queryData.getQueryMode());
BeanUtils.copyProperties(queryData, semanticQuery.getParseInfo());
ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(),
queryData.getUser().getName(), queryData.getParseId());
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
if (!parseInfo.getQueryMode().equals(DslQuery.QUERY_MODE)) {
if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {
parseInfo.setDimensions(queryData.getDimensions());
}
if (CollectionUtils.isNotEmpty(queryData.getMetrics())) {
parseInfo.setMetrics(queryData.getMetrics());
}
if (CollectionUtils.isNotEmpty(queryData.getDimensionFilters())) {
parseInfo.setDimensionFilters(queryData.getDimensionFilters());
}
}
if (Objects.nonNull(queryData.getDateInfo())) {
parseInfo.setDateInfo(queryData.getDateInfo());
}
if (parseInfo.getQueryMode().equals(DslQuery.QUERY_MODE)
&& CollectionUtils.isNotEmpty(queryData.getDimensionFilters())) {
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
LLMResp llmResp = dslParseResult.getLlmResp();
String correctorSql = llmResp.getCorrectorSql();
log.info("correctorSql before replacing:{}", correctorSql);
for (QueryFilter dslQueryFilter : queryData.getDimensionFilters()) {
for (QueryFilter queryFilter : parseInfo.getDimensionFilters()) {
if (dslQueryFilter.getBizName().equals(queryFilter.getBizName())) {
Map<String, String> map = new HashMap<>();
map.put(queryFilter.getValue().toString(), dslQueryFilter.getValue().toString());
filedNameToValueMap.put(dslQueryFilter.getBizName(), map);
break;
}
}
}
log.info("filedNameToValueMap:{}", filedNameToValueMap);
correctorSql = SqlParserUpdateHelper.replaceValue(correctorSql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", correctorSql);
llmResp.setCorrectorSql(correctorSql);
dslParseResult.setLlmResp(llmResp);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, dslParseResult);
parseInfo.setProperties(properties);
}
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
semanticQuery.setParseInfo(parseInfo);
QueryResult queryResult = semanticQuery.execute(user);
queryResult.setChatContext(semanticQuery.getParseInfo());
return queryResult;
@@ -282,8 +327,6 @@ public class QueryServiceImpl implements QueryService {
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
com.tencent.supersonic.semantic.query.service.QueryService queryService =
ContextUtils.getBean(com.tencent.supersonic.semantic.query.service.QueryService.class);
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateConf = new DateConf();
@@ -307,7 +350,8 @@ public class QueryServiceImpl implements QueryService {
dimensionFilters.add(dimensionFilter);
queryStructReq.setDimensionFilters(dimensionFilters);
}
QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStructWithAuth(queryStructReq, user);
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticLayer.queryByStruct(queryStructReq, user);
Set<String> dimensionValues = new HashSet<>();
queryResultWithSchemaResp.getResultList().removeIf(o -> {
if (dimensionValues.contains(o.get(dimensionValueReq.getBizName()))) {

451
chat/core/src/main/python/few_shot_example/sql_exampler.py
View File

@@ -1,348 +1,371 @@
examplars= [
{ "current_date":"2020-12-01",
"table_name":"内容库产品",
"fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question":"比较jackjchen和robinlee在内容库的访问次数",
"prior_schema_links":"""['jackjchen'->用户名, 'robinlee'->用户名]""",
examplars = [
{
"current_date": "2020-12-01",
"table_name": "内容库产品",
"fields_list": """["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question": "比较jackjchen和robinlee在内容库的访问次数",
"prior_schema_links": """['jackjchen'->用户名, 'robinlee'->用户名]""",
"analysis": """让我们一步一步地思考。在问题“比较jackjchen和robinlee在内容库的访问次数“中，我们被问：
“比较jackjchen和robinlee”，所以我们需要column=[用户名]
”内容库的访问次数“，所以我们需要column=[访问次数]
基于table和columns，可能的cell values 是 = ['jackjchen', 'robinlee']。""",
"schema_links":"""["用户名", "访问次数", "'jackjchen'", "'robinlee'"]""",
"sql":"""select 用户名, 访问次数 from 内容库产品 where 用户名 in ('jackjchen', 'robinlee') and 数据日期 = '2020-12-01' """
},
{ "current_date":"2022-11-06",
"table_name":"内容库产品",
"fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question":"内容库近12个月访问人数 按部门",
"prior_schema_links":"""[]""",
"schema_links": """["用户名", "访问次数", "'jackjchen'", "'robinlee'"]""",
"sql": """select 用户名, 访问次数 from 内容库产品 where 用户名 in ('jackjchen', 'robinlee') and 数据日期 = '2020-12-01' """,
},
{
"current_date": "2022-11-06",
"table_name": "内容库产品",
"fields_list": """["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question": "内容库近12个月访问人数 按部门",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库近12个月访问人数 按部门“中，我们被问：
”内容库近12个月“，所以我们需要column=[数据日期]
“访问人数”，所以我们需要column=[访问人数]
”按部门“，所以我们需要column=[部门]
基于table和columns，可能的cell values 是 = [12]。""",
"schema_links":"""["访问人数", "部门", "数据日期", 12]""",
"sql":"""select 部门, 数据日期, 访问人数 from 内容库产品 where datediff('month', 数据日期, '2022-11-06') <= 12 """
},
{ "current_date":"2023-04-21",
"table_name":"内容库产品",
"fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question":"内容库美术部、技术研发部的访问时长",
"prior_schema_links":"""['美术部'->部门, '技术研发部'->部门]""",
"schema_links": """["访问人数", "部门", "数据日期", 12]""",
"sql": """select 部门, 数据日期, 访问人数 from 内容库产品 where datediff('month', 数据日期, '2022-11-06') <= 12 """,
},
{
"current_date": "2023-04-21",
"table_name": "内容库产品",
"fields_list": """["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question": "内容库美术部、技术研发部的访问时长",
"prior_schema_links": """['美术部'->部门, '技术研发部'->部门]""",
"analysis": """让我们一步一步地思考。在问题“内容库美术部、技术研发部的访问时长“中，我们被问：
“访问时长”，所以我们需要column=[访问时长]
”内容库美术部、技术研发部“，所以我们需要column=[部门]
基于table和columns，可能的cell values 是 = ['美术部', '技术研发部']。""",
"schema_links":"""["访问时长", "部门", "'美术部'", "'技术研发部'"]""",
"sql":"""select 部门, 访问时长 from 内容库产品 where 部门 in ('美术部', '技术研发部') and 数据日期 = '2023-04-21' """
},
{ "current_date":"2023-08-21",
"table_name":"严选",
"fields_list":"""["严选版权归属系", "付费模式", "结算播放份额", "付费用户结算播放份额", "数据日期"]""",
"question":"近3天海田飞系MPPM结算播放份额",
"prior_schema_links":"""['海田飞系'->严选版权归属系]""",
"schema_links": """["访问时长", "部门", "'美术部'", "'技术研发部'"]""",
"sql": """select 部门, 访问时长 from 内容库产品 where 部门 in ('美术部', '技术研发部') and 数据日期 = '2023-04-21' """,
},
{
"current_date": "2023-08-21",
"table_name": "严选",
"fields_list": """["严选版权归属系", "付费模式", "结算播放份额", "付费用户结算播放份额", "数据日期"]""",
"question": "近3天海田飞系MPPM结算播放份额",
"prior_schema_links": """['海田飞系'->严选版权归属系]""",
"analysis": """让我们一步一步地思考。在问题“近3天海田飞系MPPM结算播放份额“中，我们被问：
“MPPM结算播放份额”，所以我们需要column=[结算播放份额]
”海田飞系“，所以我们需要column=[严选版权归属系]
”近3天“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = ['海田飞系', 3]。""",
"schema_links":"""["结算播放份额", "严选版权归属系", "数据日期", "'海田飞系'", 3]""",
"sql":"""select 严选版权归属系, 结算播放份额 from 严选 where 严选版权归属系 = '海田飞系' and datediff('day', 数据日期, '2023-08-21') <= 3 """
},
{ "current_date":"2023-05-22",
"table_name":"歌曲库",
"fields_list":"""["是否潮流人歌曲", "C音歌曲ID", "C音歌曲MID", "歌曲名", "歌曲版本", "语种", "歌曲类型", "翻唱类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "结算播放量", "运营播放量", "付费用户结算播放量", "历史累计结算播放量", "运营搜播量", "结算搜播量", "运营完播量", "运营推播量", "近7日复播率", "日均搜播量", "数据日期"]""",
"question":"对比近7天翻唱版和纯音乐的歌曲播放量",
"prior_schema_links":"""['纯音乐'->语种, '翻唱版'->歌曲版本]""",
"schema_links": """["结算播放份额", "严选版权归属系", "数据日期", "'海田飞系'", 3]""",
"sql": """select 严选版权归属系, 结算播放份额 from 严选 where 严选版权归属系 = '海田飞系' and datediff('day', 数据日期, '2023-08-21') <= 3 """,
},
{
"current_date": "2023-05-22",
"table_name": "歌曲库",
"fields_list": """["是否潮流人歌曲", "C音歌曲ID", "C音歌曲MID", "歌曲名", "歌曲版本", "语种", "歌曲类型", "翻唱类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "结算播放量", "运营播放量", "付费用户结算播放量", "历史累计结算播放量", "运营搜播量", "结算搜播量", "运营完播量", "运营推播量", "近7日复播率", "日均搜播量", "数据日期"]""",
"question": "对比近7天翻唱版和纯音乐的歌曲播放量",
"prior_schema_links": """['纯音乐'->语种, '翻唱版'->歌曲版本]""",
"analysis": """让我们一步一步地思考。在问题“对比近3天翻唱版和纯音乐的歌曲播放量“中，我们被问：
“歌曲播放量”，所以我们需要column=[结算播放量]
”翻唱版“，所以我们需要column=[歌曲版本]
”和纯音乐的歌曲“，所以我们需要column=[语种]
”近7天“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = ['翻唱版', '纯音乐', 7]。""",
"schema_links":"""["结算播放量", "歌曲版本", "语种", "数据日期", "'翻唱版'", "'纯音乐'", 7]""",
"sql":"""select 歌曲版本, 语种, 结算播放量 from 歌曲库 where 歌曲版本 = '翻唱版' and 语种 = '纯音乐' and datediff('day', 数据日期, '2023-05-22') <= 7 """
},
{ "current_date":"2023-05-31",
"table_name":"艺人库",
"fields_list":"""["上下架状态", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "活跃区域", "年龄", "歌手才能", "歌手风格", "粉丝数", "潮音粉丝数", "超声波粉丝数", "推博粉丝数", "超声波歌曲数", "在架歌曲数", "超声波分享数", "独占歌曲数", "超声波在架歌曲评论数", "有播放量歌曲数", "数据日期"]""",
"question":"对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数",
"prior_schema_links":"""['1527896'->MPPM歌手ID, '1565463'->MPPM歌手ID, '2141459'->MPPM歌手ID]""",
"schema_links": """["结算播放量", "歌曲版本", "语种", "数据日期", "'翻唱版'", "'纯音乐'", 7]""",
"sql": """select 歌曲版本, 语种, 结算播放量 from 歌曲库 where 歌曲版本 = '翻唱版' and 语种 = '纯音乐' and datediff('day', 数据日期, '2023-05-22') <= 7 """,
},
{
"current_date": "2023-05-31",
"table_name": "艺人库",
"fields_list": """["上下架状态", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "活跃区域", "年龄", "歌手才能", "歌手风格", "粉丝数", "潮音粉丝数", "超声波粉丝数", "推博粉丝数", "超声波歌曲数", "在架歌曲数", "超声波分享数", "独占歌曲数", "超声波在架歌曲评论数", "有播放量歌曲数", "数据日期"]""",
"question": "对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数",
"prior_schema_links": """['1527896'->MPPM歌手ID, '1565463'->MPPM歌手ID, '2141459'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数“中，我们被问：
“粉丝数”，所以我们需要column=[粉丝数]
”陈拙悬、孟梅琦、赖媚韵“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = ['陈拙悬', '孟梅琦', '赖媚韵']。""",
"schema_links":"""["粉丝数", "歌手名", "'陈拙悬'", "'孟梅琦'", "'赖媚韵'"]""",
"sql":"""select 歌手名, 粉丝数 from 艺人库 where 歌手名 in ('陈拙悬', '孟梅琦', '赖媚韵') and 数据日期 = '2023-05-31' """
},
{ "current_date":"2023-07-31",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"播放量大于1万的歌曲有多少",
"prior_schema_links":"""[]""",
"schema_links": """["粉丝数", "歌手名", "'陈拙悬'", "'孟梅琦'", "'赖媚韵'"]""",
"sql": """select 歌手名, 粉丝数 from 艺人库 where 歌手名 in ('陈拙悬', '孟梅琦', '赖媚韵') and 数据日期 = '2023-05-31' """,
},
{
"current_date": "2023-07-31",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "播放量大于1万的歌曲有多少",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“播放量大于1万的歌曲有多少“中，我们被问：
“歌曲有多少”，所以我们需要column=[歌曲名]
”播放量大于1万的“，所以我们需要column=[结算播放量]
基于table和columns，可能的cell values 是 = [10000]。""",
"schema_links":"""["歌曲名", "结算播放量", 10000]""",
"sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 10000 and 数据日期 = '2023-07-31' """
},
{ "current_date":"2023-07-31",
"table_name":"内容库产品",
"fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question":"内容库访问时长小于1小时，且来自美术部的用户是哪些",
"prior_schema_links":"""['美术部'->部门]""",
"schema_links": """["歌曲名", "结算播放量", 10000]""",
"sql": """select 歌曲名 from 歌曲库 where 结算播放量 > 10000 and 数据日期 = '2023-07-31' """,
},
{
"current_date": "2023-07-31",
"table_name": "内容库产品",
"fields_list": """["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question": "内容库访问时长小于1小时，且来自美术部的用户是哪些",
"prior_schema_links": """['美术部'->部门]""",
"analysis": """让我们一步一步地思考。在问题“内容库访问时长小于1小时，且来自美术部的用户是哪些“中，我们被问：
“用户是哪些”，所以我们需要column=[用户名]
”美术部的“，所以我们需要column=[部门]
”访问时长小于1小时“，所以我们需要column=[访问时长]
基于table和columns，可能的cell values 是 = ['美术部', 1]。""",
"schema_links":"""["用户名", "部门", "访问时长", "'美术部'", 1]""",
"sql":"""select 用户名 from 内容库产品 where 部门 = '美术部' and 访问时长 < 1 and 数据日期 = '2023-07-31' """
},
{ "current_date":"2023-08-31",
"table_name":"内容库产品",
"fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question":"内容库pv最高的用户有哪些",
"prior_schema_links":"""[]""",
"schema_links": """["用户名", "部门", "访问时长", "'美术部'", 1]""",
"sql": """select 用户名 from 内容库产品 where 部门 = '美术部' and 访问时长 < 1 and 数据日期 = '2023-07-31' """,
},
{
"current_date": "2023-08-31",
"table_name": "内容库产品",
"fields_list": """["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question": "内容库pv最高的用户有哪些",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库pv最高的用户有哪些“中，我们被问：
“用户有哪些”，所以我们需要column=[用户名]
”pv最高的“，所以我们需要column=[访问次数]
基于table和columns，可能的cell values 是 = []。""",
"schema_links":"""["用户名", "访问次数"]""",
"sql":"""select 用户名 from 内容库产品 where 数据日期 = '2023-08-31' order by 访问次数 desc limit 10 """
},
{ "current_date":"2023-08-31",
"table_name":"艺人库",
"fields_list":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
"question":"近90天袁亚伟播放量平均值是多少",
"prior_schema_links":"""['152789226'->MPPM歌手ID]""",
"schema_links": """["用户名", "访问次数"]""",
"sql": """select 用户名 from 内容库产品 where 数据日期 = '2023-08-31' order by 访问次数 desc limit 10 """,
},
{
"current_date": "2023-08-31",
"table_name": "艺人库",
"fields_list": """["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
"question": "近90天袁亚伟播放量平均值是多少",
"prior_schema_links": """['152789226'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“近90天袁亚伟播放量平均值是多少“中，我们被问：
“播放量平均值是多少”，所以我们需要column=[结算播放量]
”袁亚伟“，所以我们需要column=[歌手名]
”近90天“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = ['袁亚伟', 90]。""",
"schema_links":"""["结算播放量", "歌手名", "数据日期", "'袁亚伟'", 90]""",
"sql":"""select avg(结算播放量) from 艺人库 where 歌手名 = '袁亚伟' and datediff('day', 数据日期, '2023-08-31') <= 90 """
},
{ "current_date":"2023-08-31",
"table_name":"艺人库",
"fields_list":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
"question":"周倩倩近7天结算播放量总和是多少",
"prior_schema_links":"""['199509'->MPPM歌手ID]""",
"schema_links": """["结算播放量", "歌手名", "数据日期", "'袁亚伟'", 90]""",
"sql": """select avg(结算播放量) from 艺人库 where 歌手名 = '袁亚伟' and datediff('day', 数据日期, '2023-08-31') <= 90 """,
},
{
"current_date": "2023-08-31",
"table_name": "艺人库",
"fields_list": """["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
"question": "周倩倩近7天结算播放量总和是多少",
"prior_schema_links": """['199509'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“周倩倩近7天结算播放量总和是多少“中，我们被问：
“结算播放量总和是多少”，所以我们需要column=[结算播放量]
”周倩倩“，所以我们需要column=[歌手名]
”近7天“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = ['周倩倩', 7]。""",
"schema_links":"""["结算播放量", "歌手名", "数据日期", "'周倩倩'", 7]""",
"sql":"""select sum(结算播放量) from 艺人库 where 歌手名 = '周倩倩' and datediff('day', 数据日期, '2023-08-31') <= 7 """
},
{ "current_date":"2023-09-14",
"table_name":"内容库产品",
"fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question":"内容库访问次数大于1k的部门是哪些",
"prior_schema_links":"""[]""",
"schema_links": """["结算播放量", "歌手名", "数据日期", "'周倩倩'", 7]""",
"sql": """select sum(结算播放量) from 艺人库 where 歌手名 = '周倩倩' and datediff('day', 数据日期, '2023-08-31') <= 7 """,
},
{
"current_date": "2023-09-14",
"table_name": "内容库产品",
"fields_list": """["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""",
"question": "内容库访问次数大于1k的部门是哪些",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库访问次数大于1k的部门是哪些“中，我们被问：
“部门是哪些”，所以我们需要column=[部门]
”访问次数大于1k的“，所以我们需要column=[访问次数]
基于table和columns，可能的cell values 是 = [1000]。""",
"schema_links":"""["部门", "访问次数", 1000]""",
"sql":"""select 部门 from 内容库产品 where 访问次数 > 1000 and 数据日期 = '2023-09-14' """
},
{ "current_date":"2023-09-18",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "MPPM歌手ID", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"陈亿训唱的所有的播放量大于20k的孤勇者有哪些",
"prior_schema_links":"""['199509'->MPPM歌手ID, '1527123'->MPPM歌曲ID]""",
"schema_links": """["部门", "访问次数", 1000]""",
"sql": """select 部门 from 内容库产品 where 访问次数 > 1000 and 数据日期 = '2023-09-14' """,
},
{
"current_date": "2023-09-18",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "MPPM歌手ID", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "陈亿训唱的所有的播放量大于20k的孤勇者有哪些",
"prior_schema_links": """['199509'->MPPM歌手ID, '1527123'->MPPM歌曲ID]""",
"analysis": """让我们一步一步地思考。在问题“陈亿训唱的所有的播放量大于20k的孤勇者有哪些“中，我们被问：
“孤勇者有哪些”，所以我们需要column=[歌曲名]
”播放量大于20k的“，所以我们需要column=[结算播放量]
”陈亿训唱的“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = [20000, '陈亿训', '孤勇者']。""",
"schema_links":"""["歌曲名", "结算播放量", "歌手名", 20000, "'陈亿训'", "'孤勇者'"]""",
"sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 20000 and 歌手名 = '陈亿训' and 歌曲名 = '孤勇者' and 数据日期 = '2023-09-18' """
},
{ "current_date":"2023-09-18",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"周洁轮去年发布的歌曲有哪些",
"prior_schema_links":"""['23109'->MPPM歌手ID]""",
"schema_links": """["歌曲名", "结算播放量", "歌手名", 20000, "'陈亿训'", "'孤勇者'"]""",
"sql": """select 歌曲名 from 歌曲库 where 结算播放量 > 20000 and 歌手名 = '陈亿训' and 歌曲名 = '孤勇者' and 数据日期 = '2023-09-18' """,
},
{
"current_date": "2023-09-18",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "周洁轮去年发布的歌曲有哪些",
"prior_schema_links": """['23109'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“周洁轮去年发布的歌曲有哪些“中，我们被问：
“歌曲有哪些”，所以我们需要column=[歌曲名]
”去年发布的“，所以我们需要column=[发布时间]
”周洁轮“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = ['周洁轮', 1]。""",
"schema_links":"""["歌曲名", "发布时间", "歌手名", 1, "'周洁轮'"]""",
"sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发布时间, '2023-09-18') <= 1 and 歌手名 = '周洁轮' and 数据日期 = '2023-09-18' """
},
{ "current_date":"2023-09-11",
"table_name":"艺人库",
"fields_list":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "签约日期", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
"question":"我想要近半年签约的播放量前十的歌手有哪些",
"prior_schema_links":"""[]""",
"schema_links": """["歌曲名", "发布时间", "歌手名", 1, "'周洁轮'"]""",
"sql": """select 歌曲名 from 歌曲库 where datediff('year', 发布时间, '2023-09-18') <= 1 and 歌手名 = '周洁轮' and 数据日期 = '2023-09-18' """,
},
{
"current_date": "2023-09-11",
"table_name": "艺人库",
"fields_list": """["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "签约日期", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""",
"question": "我想要近半年签约的播放量前十的歌手有哪些",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“我想要近半年签约的播放量前十的歌手“中，我们被问：
“歌手有哪些”，所以我们需要column=[歌手名]
”播放量前十的“，所以我们需要column=[结算播放量]
”近半年签约的“，所以我们需要column=[签约日期]
基于table和columns，可能的cell values 是 = [0.5, 10]。""",
"schema_links":"""["歌手名", "结算播放量", "签约日期", 0.5, 10]""",
"sql":"""select 歌手名 from 艺人库 where datediff('year', 签约日期, '2023-09-11') <= 0.5 and 数据日期 = '2023-09-11' order by 结算播放量 desc limit 10"""
},
{ "current_date":"2023-08-12",
"table_name":"歌曲库",
"schema_links": """["歌手名", "结算播放量", "签约日期", 0.5, 10]""",
"sql": """select 歌手名 from 艺人库 where datediff('year', 签约日期, '2023-09-11') <= 0.5 and 数据日期 = '2023-09-11' order by 结算播放量 desc limit 10""",
},
{
"current_date": "2023-08-12",
"table_name": "歌曲库",
"fields_list": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""",
"question":"最近一年发行的歌曲中，有哪些在近7天播放超过一千万的",
"prior_schema_links":"""[]""",
"question": "最近一年发行的歌曲中，有哪些在近7天播放超过一千万的",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“最近一年发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问：
“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名]
”最近一年发行的“，所以我们需要column=[发行日期]
”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量]
基于table和columns，可能的cell values 是 = [1, 10000000]""",
"schema_links":"""["歌曲名", "发行日期", "数据日期", "结算播放量", 1, 10000000]""",
"sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 1 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000"""
},
{ "current_date":"2023-08-12",
"table_name":"歌曲库",
"schema_links": """["歌曲名", "发行日期", "数据日期", "结算播放量", 1, 10000000]""",
"sql": """select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 1 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000""",
},
{
"current_date": "2023-08-12",
"table_name": "歌曲库",
"fields_list": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""",
"question":"今年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
"prior_schema_links":"""[]""",
"question": "今年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“今年以来发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问：
“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名]
”今年以来发行的“，所以我们需要column=[发行日期]
”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量]
基于table和columns，可能的cell values 是 = [0, 7, 10000000]""",
"schema_links":"""["歌曲名", "发行日期", "数据日期", "结算播放量", 0, 7, 10000000]""",
"sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 0 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000"""
},
{ "current_date":"2023-08-12",
"table_name":"歌曲库",
"schema_links": """["歌曲名", "发行日期", "数据日期", "结算播放量", 0, 7, 10000000]""",
"sql": """select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 0 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000""",
},
{
"current_date": "2023-08-12",
"table_name": "歌曲库",
"fields_list": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""",
"question":"2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
"prior_schema_links":"""['514129144'->MPPM歌曲ID]""",
"question": "2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的",
"prior_schema_links": """['514129144'->MPPM歌曲ID]""",
"analysis": """让我们一步一步地思考。在问题“2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问：
“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名]
”2023年以来发行的“，所以我们需要column=[发行日期]
”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量]
基于table和columns，可能的cell values 是 = [2023, 7, 10000000]""",
"schema_links":"""["歌曲名", "发行日期", "数据日期", "结算播放量", 2023, 7, 10000000]""",
"sql":"""select 歌曲名 from 歌曲库 where YEAR(发行日期) >= 2023 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000"""
},
{ "current_date":"2023-08-01",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"周洁轮2023年6月之后发布的歌曲有哪些",
"prior_schema_links":"""['23109'->MPPM歌手ID]""",
"schema_links": """["歌曲名", "发行日期", "数据日期", "结算播放量", 2023, 7, 10000000]""",
"sql": """select 歌曲名 from 歌曲库 where YEAR(发行日期) >= 2023 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000""",
},
{
"current_date": "2023-08-01",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "周洁轮2023年6月之后发布的歌曲有哪些",
"prior_schema_links": """['23109'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“周洁轮2023年6月之后发布的歌曲有哪些“中，我们被问：
“歌曲有哪些”，所以我们需要column=[歌曲名]
”2023年6月之后发布的“，所以我们需要column=[发布时间]
”周洁轮“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = ['周洁轮', 2023, 6]。""",
"schema_links":"""["歌曲名", "发布时间", "歌手名", "周洁轮", 2023, 6]""",
"sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2023 and MONTH(发布时间) >= 6 and 歌手名 = '周洁轮' and 数据日期 = '2023-08-01' """
},
{ "current_date":"2023-08-01",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？",
"prior_schema_links":"""['2312311'->MPPM歌手ID]""",
"schema_links": """["歌曲名", "发布时间", "歌手名", "周洁轮", 2023, 6]""",
"sql": """select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2023 and MONTH(发布时间) >= 6 and 歌手名 = '周洁轮' and 数据日期 = '2023-08-01' """,
},
{
"current_date": "2023-08-01",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？",
"prior_schema_links": """['2312311'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？“中，我们被问：
“播放量大于500W的”，所以我们需要column=[结算播放量]
”邓梓琦在2023年1月5日之后发布的“，所以我们需要column=[发布时间]
”邓梓琦“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = ['邓梓琦', 2023, 1, 5, 5000000]。""",
"schema_links":"""["结算播放量", "发布时间", "歌手名", "邓梓琦", 2023, 1, 5, 5000000]""",
"sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2023 and MONTH(发布时间) >= 1 and DAY(发布时间) >= 5 and 歌手名 = '邓梓琦' and 结算播放量 > 5000000 and 数据日期 = '2023-08-01'"""
},
{ "current_date":"2023-09-17",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"2023年6月以后，张亮英播放量大于200万的歌曲有哪些？",
"prior_schema_links":"""['45453'->MPPM歌手ID]""",
"schema_links": """["结算播放量", "发布时间", "歌手名", "邓梓琦", 2023, 1, 5, 5000000]""",
"sql": """select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2023 and MONTH(发布时间) >= 1 and DAY(发布时间) >= 5 and 歌手名 = '邓梓琦' and 结算播放量 > 5000000 and 数据日期 = '2023-08-01'""",
},
{
"current_date": "2023-09-17",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "2023年6月以后，张亮英播放量大于200万的歌曲有哪些？",
"prior_schema_links": """['45453'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“2023年6月以后，张亮英播放量大于200万的歌曲有哪些？“中，我们被问：
“播放量大于200万的”，所以我们需要column=[结算播放量]
”2023年6月以后，张亮英“，所以我们需要column=[数据日期, 歌手名]
”歌曲有哪些“，所以我们需要column=[歌曲名]
基于table和columns，可能的cell values 是 = ['张亮英', 2023, 6, 2000000]。""",
"schema_links":"""["结算播放量", "数据日期", "歌手名", "张亮英", 2023, 6, 2000000]""",
"sql":"""select 歌曲名 from 歌曲库 where YEAR(数据日期) >= 2023 and MONTH(数据日期) >= 6 and 歌手名 = '张亮英' and 结算播放量 > 2000000 """
},
{ "current_date":"2023-08-16",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些",
"prior_schema_links":"""['23109'->MPPM歌手ID]""",
"schema_links": """["结算播放量", "数据日期", "歌手名", "张亮英", 2023, 6, 2000000]""",
"sql": """select 歌曲名 from 歌曲库 where YEAR(数据日期) >= 2023 and MONTH(数据日期) >= 6 and 歌手名 = '张亮英' and 结算播放量 > 2000000 """,
},
{
"current_date": "2023-08-16",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些",
"prior_schema_links": """['23109'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些“中，我们被问：
“播放量大于20万的”，所以我们需要column=[结算播放量]
”2021年6月以后发布的“，所以我们需要column=[发布时间]
”李雨纯“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = ['李雨纯', 2021, 6, 200000]。""",
"schema_links":"""["结算播放量", "发布时间", "歌手名", "李雨纯", 2021, 6, 200000]""",
"sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2021 and MONTH(发布时间) >= 6 and 歌手名 = '李雨纯' and 结算播放量 > 200000 and 数据日期 = '2023-08-16'"""
},
{ "current_date":"2023-08-16",
"table_name":"歌曲库",
"fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question":"刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些",
"prior_schema_links":"""['4234234'->MPPM歌手ID]""",
"schema_links": """["结算播放量", "发布时间", "歌手名", "李雨纯", 2021, 6, 200000]""",
"sql": """select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2021 and MONTH(发布时间) >= 6 and 歌手名 = '李雨纯' and 结算播放量 > 200000 and 数据日期 = '2023-08-16'""",
},
{
"current_date": "2023-08-16",
"table_name": "歌曲库",
"fields_list": """["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""",
"question": "刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些",
"prior_schema_links": """['4234234'->MPPM歌手ID]""",
"analysis": """让我们一步一步地思考。在问题“刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些“中，我们被问：
“播放量大于20万的”，所以我们需要column=[结算播放量]
”1992年4月2日到2020年5月2日之间发布的“，所以我们需要column=[发布时间]
”刘锝桦“，所以我们需要column=[歌手名]
基于table和columns，可能的cell values 是 = ['刘锝桦', 1992, 4, 2, 2020, 5, 2, 200000]。""",
"schema_links":"""["结算播放量", "发布时间", "歌手名", "刘锝桦", 1992, 4, 2, 2020, 5, 2, 200000]""",
"sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 1992 and MONTH(发布时间) >= 4 and DAY(发布时间) >= 2 and YEAR(发布时间) <= 2020 and MONTH(发布时间) <= 5 and DAY(发布时间) <= 2 and 歌手名 = '刘锝桦' and 结算播放量 > 200000 and 数据日期 = '2023-08-16'"""
},
"schema_links": """["结算播放量", "发布时间", "歌手名", "刘锝桦", 1992, 4, 2, 2020, 5, 2, 200000]""",
"sql": """select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 1992 and MONTH(发布时间) >= 4 and DAY(发布时间) >= 2 and YEAR(发布时间) <= 2020 and MONTH(发布时间) <= 5 and DAY(发布时间) <= 2 and 歌手名 = '刘锝桦' and 结算播放量 > 200000 and 数据日期 = '2023-08-16'""",
},
{
"current_date":"2023-09-04",
"table_name":"内容库产品",
"fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question":"内容库近30天访问次数的平均数",
"prior_schema_links":"""[]""",
"current_date": "2023-09-04",
"table_name": "内容库产品",
"fields_list": """["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question": "内容库近30天访问次数的平均数",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库近30天访问次数的平均数“中，我们被问：
“访问次数的平均数”，所以我们需要column=[访问次数]
”内容库近30天“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = [30]。""",
"schema_links":"""["访问次数", "数据日期", 30]""",
"sql":"""select avg(访问次数) from 内容库产品 where datediff('day', 数据日期, '2023-09-04') <= 30 """
},
"schema_links": """["访问次数", "数据日期", 30]""",
"sql": """select avg(访问次数) from 内容库产品 where datediff('day', 数据日期, '2023-09-04') <= 30 """,
},
{
"current_date":"2023-09-04",
"table_name":"内容库产品",
"fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question":"内容库近半年哪个月的访问次数汇总最高",
"prior_schema_links":"""[]""",
"current_date": "2023-09-04",
"table_name": "内容库产品",
"fields_list": """["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question": "内容库近半年哪个月的访问次数汇总最高",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库近半年哪个月的访问次数汇总最高“中，我们被问：
“访问次数汇总最高”，所以我们需要column=[访问次数]
”内容库近半年“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = [0.5]。""",
"schema_links":"""["访问次数", "数据日期", 0.5]""",
"sql":"""select MONTH(数据日期), sum(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期) order by sum(访问次数) desc limit 1 """
},
"schema_links": """["访问次数", "数据日期", 0.5]""",
"sql": """select MONTH(数据日期), sum(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期) order by sum(访问次数) desc limit 1 """,
},
{
"current_date":"2023-09-04",
"table_name":"内容库产品",
"fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question":"内容库近半年每个月的平均访问次数",
"prior_schema_links":"""[]""",
"current_date": "2023-09-04",
"table_name": "内容库产品",
"fields_list": """["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question": "内容库近半年每个月的平均访问次数",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库近半年每个月的平均访问次数“中，我们被问：
“每个月的平均访问次数”，所以我们需要column=[访问次数]
”内容库近半年“，所以我们需要column=[数据日期]
基于table和columns，可能的cell values 是 = [0.5]。""",
"schema_links":"""["访问次数", "数据日期", 0.5]""",
"sql":"""select MONTH(数据日期), avg(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期) """
},
"schema_links": """["访问次数", "数据日期", 0.5]""",
"sql": """select MONTH(数据日期), avg(访问次数) from 内容库产品 where datediff('year', 数据日期, '2023-09-04') <= 0.5 group by MONTH(数据日期) """,
},
{
"current_date":"2023-09-10",
"table_name":"内容库产品",
"fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question":"内容库 按部门统计访问次数 top10 的部门",
"prior_schema_links":"""[]""",
"current_date": "2023-09-10",
"table_name": "内容库产品",
"fields_list": """["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""",
"question": "内容库 按部门统计访问次数 top10 的部门",
"prior_schema_links": """[]""",
"analysis": """让我们一步一步地思考。在问题“内容库 按部门统计访问次数 top10 的部门“中，我们被问：
“访问次数 top10 的部门”，所以我们需要column=[访问次数]
”内容库 按部门统计“，所以我们需要column=[部门]
基于table和columns，可能的cell values 是 = [10]。""",
"schema_links":"""["访问次数", "部门", 10]""",
"sql":"""select 部门, sum(访问次数) from 内容库产品 group by 部门 order by sum(访问次数) desc limit 10 """
}
"schema_links": """["访问次数", "部门", 10]""",
"sql": """select 部门, sum(访问次数) from 内容库产品 group by 部门 order by sum(访问次数) desc limit 10 """,
},
]

35
chat/core/src/main/python/plugin_call/prompt_construct.py
View File

@@ -23,6 +23,7 @@ def construct_plugin_prompt(tool_config):
prompt += example + "\n"
return prompt
def construct_plugin_pool_prompt(tool_config_list):
tool_explain_list = []
for tool_config in tool_config_list:
@@ -35,15 +36,20 @@ def construct_plugin_pool_prompt(tool_config_list):
def construct_task_prompt(query_text, tool_explain_list_str):
instruction = """问题为:{query_text}\n请根据问题和工具的描述，选择对应的工具，完成任务。请注意，只能选择1个工具。请一步一步地分析选择工具的原因(每个工具的【工具适用问题示例】是选择的重要参考依据)，并给出最终选择，输出格式为json,key为’分析过程‘, ’选择工具‘""".format(query_text=query_text)
instruction = """问题为:{query_text}\n请根据问题和工具的描述，选择对应的工具，完成任务。请注意，只能选择1个工具。请一步一步地分析选择工具的原因(每个工具的【工具适用问题示例】是选择的重要参考依据)，并给出最终选择，输出格式为json,key为’分析过程‘, ’选择工具‘""".format(
query_text=query_text
)
prompt = "工具选择如下:\n\n{tool_explain_list_str}\n\n【任务说明】\n{instruction}".format(instruction=instruction, tool_explain_list_str=tool_explain_list_str)
prompt = "工具选择如下:\n\n{tool_explain_list_str}\n\n【任务说明】\n{instruction}".format(
instruction=instruction, tool_explain_list_str=tool_explain_list_str
)
return prompt
def plugin_selection_output_parse(llm_output: str)-> Union[Mapping[str, str], None]:
def plugin_selection_output_parse(llm_output: str) -> Union[Mapping[str, str], None]:
try:
pattern = r'\{[^{}]+\}'
pattern = r"\{[^{}]+\}"
find_result = re.findall(pattern, llm_output)
result = find_result[0].strip()
@@ -52,12 +58,13 @@ def plugin_selection_output_parse(llm_output: str)-> Union[Mapping[str, str], No
result_dict = json.loads(result)
print("result_dict: ", result_dict)
key_mapping = {
"分析过程":"analysis",
"选择工具":"toolSelection"
}
key_mapping = {"分析过程": "analysis", "选择工具": "toolSelection"}
converted_result_dict = {key_mapping[key]: value for key, value in result_dict.items() if key in key_mapping}
converted_result_dict = {
key_mapping[key]: value
for key, value in result_dict.items()
if key in key_mapping
}
except Exception as e:
print(e)
@@ -65,7 +72,10 @@ def plugin_selection_output_parse(llm_output: str)-> Union[Mapping[str, str], No
return converted_result_dict
def plugins_config_format_convert(plugin_config_list: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]:
def plugins_config_format_convert(
plugin_config_list: List[Mapping[str, Any]]
) -> List[Mapping[str, Any]]:
plugin_config_list_new = []
for plugin_config in plugin_config_list:
plugin_config_new = dict()
@@ -75,7 +85,9 @@ def plugins_config_format_convert(plugin_config_list: List[Mapping[str, Any]]) -
parameters = plugin_config["parameters"]
examples_str = "\n".join(examples)
description_new = """{plugin_desc}\n\n例如能够处理如下问题:\n{examples_str}""".format(plugin_desc=description, examples_str=examples_str)
description_new = """{plugin_desc}\n\n例如能够处理如下问题:\n{examples_str}""".format(
plugin_desc=description, examples_str=examples_str
)
plugin_config_new["name"] = name
plugin_config_new["description"] = description_new
@@ -84,4 +96,3 @@ def plugins_config_format_convert(plugin_config_list: List[Mapping[str, Any]]) -
plugin_config_list_new.append(plugin_config_new)
return plugin_config_list_new

12
chat/core/src/main/python/plugin_call/run.py
View File

@@ -10,11 +10,18 @@ import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from plugin_call.prompt_construct import construct_plugin_pool_prompt, construct_task_prompt, plugin_selection_output_parse, plugins_config_format_convert
from plugin_call.prompt_construct import (
construct_plugin_pool_prompt,
construct_task_prompt,
plugin_selection_output_parse,
plugins_config_format_convert,
)
from util.llm_instance import llm
def plugin_selection_run(query_text: str, plugin_configs: List[Mapping[str, Any]])-> Union[Mapping[str, str], None]:
def plugin_selection_run(
query_text: str, plugin_configs: List[Mapping[str, Any]]
) -> Union[Mapping[str, str], None]:
tools_prompt = construct_plugin_pool_prompt(plugin_configs)
@@ -23,4 +30,3 @@ def plugin_selection_run(query_text: str, plugin_configs: List[Mapping[str, Any]
parsed_output = plugin_selection_output_parse(llm_output)
return parsed_output

75
chat/core/src/main/python/preset_retrieval/preset_query_db.py
View File

@@ -11,7 +11,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
def get_ids(documents:List[str]) -> List[str]:
def get_ids(documents: List[str]) -> List[str]:
ids = []
for doc in documents:
ids.append(str(uuid.uuid5(uuid.NAMESPACE_URL, doc)))
@@ -19,25 +19,23 @@ def get_ids(documents:List[str]) -> List[str]:
return ids
def add2preset_query_collection(collection:Collection,
preset_queries:List[str],
preset_query_ids:List[str]
) -> None:
def add2preset_query_collection(
collection: Collection, preset_queries: List[str], preset_query_ids: List[str]
) -> None:
collection.add(documents=preset_queries,
ids=preset_query_ids)
collection.add(documents=preset_queries, ids=preset_query_ids)
def update_preset_query_collection(collection:Collection,
preset_queries:List[str],
preset_query_ids:List[str]
) -> None:
def update_preset_query_collection(
collection: Collection, preset_queries: List[str], preset_query_ids: List[str]
) -> None:
collection.update(documents=preset_queries,
ids=preset_query_ids)
collection.update(documents=preset_queries, ids=preset_query_ids)
def query2preset_query_collection(collection:Collection, query_texts:List[str], n_results:int=10):
def query2preset_query_collection(
collection: Collection, query_texts: List[str], n_results: int = 10
):
collection_cnt = collection.count()
min_n_results = 10
min_n_results = min(collection_cnt, min_n_results)
@@ -56,12 +54,13 @@ def query2preset_query_collection(collection:Collection, query_texts:List[str],
return res
def parse_retrieval_preset_query(res:List[Mapping[str, Any]]):
parsed_res = [[] for _ in range(0, len(res['ids']))]
retrieval_ids = res['ids']
retrieval_distances = res['distances']
retrieval_sentences = res['documents']
def parse_retrieval_preset_query(res: List[Mapping[str, Any]]):
parsed_res = [[] for _ in range(0, len(res["ids"]))]
retrieval_ids = res["ids"]
retrieval_distances = res["distances"]
retrieval_sentences = res["documents"]
for query_idx in range(0, len(retrieval_ids)):
id_ls = retrieval_ids[query_idx]
@@ -73,43 +72,41 @@ def parse_retrieval_preset_query(res:List[Mapping[str, Any]]):
distance = distance_ls[idx]
sentence = sentence_ls[idx]
parsed_res[query_idx].append({
'id': id,
'distance': distance,
'presetQuery': sentence
})
parsed_res[query_idx].append(
{"id": id, "distance": distance, "presetQuery": sentence}
)
return parsed_res
def preset_query_retrieval_format(query_list:List[str], retrieval_list:List[Mapping[str, Any]]):
def preset_query_retrieval_format(
query_list: List[str], retrieval_list: List[Mapping[str, Any]]
):
res = []
for query_idx in range(0, len(query_list)):
query = query_list[query_idx]
retrieval = retrieval_list[query_idx]
res.append({
'query': query,
'retrieval': retrieval
})
res.append({"query": query, "retrieval": retrieval})
return res
def empty_preset_query_collection(collection:Collection) -> None:
def empty_preset_query_collection(collection: Collection) -> None:
collection.delete()
def delete_preset_query_by_ids(collection:Collection, preset_query_ids:List[str]) -> None:
def delete_preset_query_by_ids(
collection: Collection, preset_query_ids: List[str]
) -> None:
collection.delete(ids=preset_query_ids)
def get_preset_query_by_ids(collection:Collection, preset_query_ids:List[str]):
def get_preset_query_by_ids(collection: Collection, preset_query_ids: List[str]):
res = collection.get(ids=preset_query_ids)
return res
def preset_query_collection_size(collection:Collection) -> int:
def preset_query_collection_size(collection: Collection) -> int:
return collection.count()

37
chat/core/src/main/python/preset_retrieval/run.py
View File

@@ -13,9 +13,15 @@ from chromadb.api import Collection, Documents, Embeddings
from langchain.llms import OpenAI
from preset_query_db import (get_ids, add2preset_query_collection,
query2preset_query_collection, parse_retrieval_preset_query,
preset_query_retrieval_format, empty_preset_query_collection, preset_query_collection_size)
from preset_query_db import (
get_ids,
add2preset_query_collection,
query2preset_query_collection,
parse_retrieval_preset_query,
preset_query_retrieval_format,
empty_preset_query_collection,
preset_query_collection_size,
)
from util.text2vec import Text2VecEmbeddingFunction
@@ -25,22 +31,27 @@ from util.chromadb_instance import client
emb_func = Text2VecEmbeddingFunction()
collection = client.get_or_create_collection(name=PRESET_QUERY_COLLECTION_NAME,
embedding_function=emb_func,
metadata={"hnsw:space": "cosine"}
) # Get a collection object from an existing collection, by name. If it doesn't exist, create it.
collection = client.get_or_create_collection(
name=PRESET_QUERY_COLLECTION_NAME,
embedding_function=emb_func,
metadata={"hnsw:space": "cosine"},
) # Get a collection object from an existing collection, by name. If it doesn't exist, create it.
print("init_preset_query_collection_size: ", preset_query_collection_size(collection))
def preset_query_retrieval_run(collection:Collection, query_texts_list:List[str], n_results:int=5):
retrieval_res = query2preset_query_collection(collection=collection,
query_texts=query_texts_list,
n_results=n_results)
def preset_query_retrieval_run(
collection: Collection, query_texts_list: List[str], n_results: int = 5
):
retrieval_res = query2preset_query_collection(
collection=collection, query_texts=query_texts_list, n_results=n_results
)
parsed_retrieval_res = parse_retrieval_preset_query(retrieval_res)
parsed_retrieval_res_format = preset_query_retrieval_format(query_texts_list, parsed_retrieval_res)
parsed_retrieval_res_format = preset_query_retrieval_format(
query_texts_list, parsed_retrieval_res
)
print('parsed_retrieval_res_format: ', parsed_retrieval_res_format)
print("parsed_retrieval_res_format: ", parsed_retrieval_res_format)
return parsed_retrieval_res_format

14
chat/core/src/main/python/run_config.py
View File

@@ -11,7 +11,7 @@ OPENAI_API_KEY = "YOUR_API_KEY"
TEMPERATURE = 0.0
CHROMA_DB_PERSIST_DIR = 'chm_db'
CHROMA_DB_PERSIST_DIR = "chm_db"
PRESET_QUERY_COLLECTION_NAME = "preset_query_collection"
TEXT2DSL_COLLECTION_NAME = "text2dsl_collection"
TEXT2DSL_FEW_SHOTS_EXAMPLE_NUM = 15
@@ -21,9 +21,9 @@ CHROMA_DB_PERSIST_PATH = os.path.join(PROJECT_DIR_PATH, CHROMA_DB_PERSIST_DIR)
HF_TEXT2VEC_MODEL_NAME = "GanymedeNil/text2vec-large-chinese"
if __name__ == '__main__':
print('PROJECT_DIR_PATH: ', PROJECT_DIR_PATH)
print('EMB_MODEL_PATH: ', HF_TEXT2VEC_MODEL_NAME)
print('CHROMA_DB_PERSIST_PATH: ', CHROMA_DB_PERSIST_PATH)
print('LLMPARSER_HOST: ', LLMPARSER_HOST)
print('LLMPARSER_PORT: ', LLMPARSER_PORT)
if __name__ == "__main__":
print("PROJECT_DIR_PATH: ", PROJECT_DIR_PATH)
print("EMB_MODEL_PATH: ", HF_TEXT2VEC_MODEL_NAME)
print("CHROMA_DB_PERSIST_PATH: ", CHROMA_DB_PERSIST_PATH)
print("LLMPARSER_HOST: ", LLMPARSER_HOST)
print("LLMPARSER_PORT: ", LLMPARSER_PORT)

60
chat/core/src/main/python/sql/constructor.py
View File

@@ -22,20 +22,34 @@ from util.text2vec import Text2VecEmbeddingFunction, hg_embedding
from util.chromadb_instance import client as chromadb_client, empty_chroma_collection_2
from run_config import TEXT2DSL_COLLECTION_NAME, TEXT2DSL_FEW_SHOTS_EXAMPLE_NUM
def reload_sql_example_collection(vectorstore:Chroma,
sql_examplars:List[Mapping[str, str]],
sql_example_selector:SemanticSimilarityExampleSelector,
example_nums:int
):
def reload_sql_example_collection(
vectorstore: Chroma,
sql_examplars: List[Mapping[str, str]],
sql_example_selector: SemanticSimilarityExampleSelector,
example_nums: int,
):
print("original sql_examples_collection size:", vectorstore._collection.count())
new_collection = empty_chroma_collection_2(collection=vectorstore._collection)
vectorstore._collection = new_collection
print("emptied sql_examples_collection size:", vectorstore._collection.count())
sql_example_selector = SemanticSimilarityExampleSelector(vectorstore=sql_examples_vectorstore, k=example_nums,
input_keys=["question"],
example_keys=["table_name", "fields_list", "prior_schema_links", "question", "analysis", "schema_links", "current_date", "sql"])
sql_example_selector = SemanticSimilarityExampleSelector(
vectorstore=sql_examples_vectorstore,
k=example_nums,
input_keys=["question"],
example_keys=[
"table_name",
"fields_list",
"prior_schema_links",
"question",
"analysis",
"schema_links",
"current_date",
"sql",
],
)
for example in sql_examplars:
sql_example_selector.add_example(example)
@@ -45,20 +59,36 @@ def reload_sql_example_collection(vectorstore:Chroma,
return vectorstore, sql_example_selector
sql_examples_vectorstore = Chroma(collection_name=TEXT2DSL_COLLECTION_NAME,
embedding_function=hg_embedding,
client=chromadb_client)
sql_examples_vectorstore = Chroma(
collection_name=TEXT2DSL_COLLECTION_NAME,
embedding_function=hg_embedding,
client=chromadb_client,
)
example_nums = TEXT2DSL_FEW_SHOTS_EXAMPLE_NUM
sql_example_selector = SemanticSimilarityExampleSelector(vectorstore=sql_examples_vectorstore, k=example_nums,
input_keys=["question"],
example_keys=["table_name", "fields_list", "prior_schema_links", "question", "analysis", "schema_links", "current_date", "sql"])
sql_example_selector = SemanticSimilarityExampleSelector(
vectorstore=sql_examples_vectorstore,
k=example_nums,
input_keys=["question"],
example_keys=[
"table_name",
"fields_list",
"prior_schema_links",
"question",
"analysis",
"schema_links",
"current_date",
"sql",
],
)
if sql_examples_vectorstore._collection.count() > 0:
print("examples already in sql_vectorstore")
print("init sql_vectorstore size:", sql_examples_vectorstore._collection.count())
print("sql_examplars size:", len(sql_examplars))
sql_examples_vectorstore, sql_example_selector = reload_sql_example_collection(sql_examples_vectorstore, sql_examplars, sql_example_selector, example_nums)
sql_examples_vectorstore, sql_example_selector = reload_sql_example_collection(
sql_examples_vectorstore, sql_examplars, sql_example_selector, example_nums
)
print("added sql_vectorstore size:", sql_examples_vectorstore._collection.count())

26
chat/core/src/main/python/sql/examples_reload_run.py
View File

@@ -13,17 +13,31 @@ from few_shot_example.sql_exampler import examplars as sql_examplars
from run_config import LLMPARSER_HOST, LLMPARSER_PORT
def text2dsl_setting_update(llm_parser_host:str, llm_parser_port:str,
sql_examplars:List[Mapping[str, str]], example_nums:int, is_shortcut:bool):
def text2dsl_setting_update(
llm_parser_host: str,
llm_parser_port: str,
sql_examplars: List[Mapping[str, str]],
example_nums: int,
is_shortcut: bool,
):
url = f"http://{llm_parser_host}:{llm_parser_port}/query2sql_setting_update/"
print("url: ", url)
payload = {"sqlExamplars":sql_examplars, "exampleNums":example_nums, "isShortcut":is_shortcut}
headers = {'content-type': 'application/json'}
payload = {
"sqlExamplars": sql_examplars,
"exampleNums": example_nums,
"isShortcut": is_shortcut,
}
headers = {"content-type": "application/json"}
response = requests.post(url, data=json.dumps(payload), headers=headers)
print(response.text)
if __name__ == "__main__":
text2dsl_setting_update(LLMPARSER_HOST, LLMPARSER_PORT,
sql_examplars, TEXT2DSL_FEW_SHOTS_EXAMPLE_NUM, TEXT2DSL_IS_SHORTCUT)
text2dsl_setting_update(
LLMPARSER_HOST,
LLMPARSER_PORT,
sql_examplars,
TEXT2DSL_FEW_SHOTS_EXAMPLE_NUM,
TEXT2DSL_IS_SHORTCUT,
)

13
chat/core/src/main/python/sql/output_parser.py
View File

@@ -1,21 +1,25 @@
# -*- coding:utf-8 -*-
import re
def schema_link_parse(schema_link_output):
try:
schema_link_output = schema_link_output.strip()
pattern = r'Schema_links:(.*)'
schema_link_output = re.findall(pattern, schema_link_output, re.DOTALL)[0].strip()
pattern = r"Schema_links:(.*)"
schema_link_output = re.findall(pattern, schema_link_output, re.DOTALL)[
0
].strip()
except Exception as e:
print(e)
schema_link_output = None
return schema_link_output
def combo_schema_link_parse(schema_linking_sql_combo_output: str):
try:
schema_linking_sql_combo_output = schema_linking_sql_combo_output.strip()
pattern = r'Schema_links:(\[.*?\])'
pattern = r"Schema_links:(\[.*?\])"
schema_links_match = re.search(pattern, schema_linking_sql_combo_output)
if schema_links_match:
@@ -28,10 +32,11 @@ def combo_schema_link_parse(schema_linking_sql_combo_output: str):
return schema_links
def combo_sql_parse(schema_linking_sql_combo_output: str):
try:
schema_linking_sql_combo_output = schema_linking_sql_combo_output.strip()
pattern = r'SQL:(.*)'
pattern = r"SQL:(.*)"
sql_match = re.search(pattern, schema_linking_sql_combo_output)
if sql_match:

152
chat/core/src/main/python/sql/prompt_maker.py
View File

@@ -11,17 +11,31 @@ from langchain.prompts.few_shot import FewShotPromptTemplate
from langchain.prompts.example_selector import SemanticSimilarityExampleSelector
def schema_linking_exampler(user_query: str,
domain_name: str,
fields_list: List[str],
prior_schema_links: Mapping[str,str],
example_selector: SemanticSimilarityExampleSelector,
) -> str:
def schema_linking_exampler(
user_query: str,
domain_name: str,
fields_list: List[str],
prior_schema_links: Mapping[str, str],
example_selector: SemanticSimilarityExampleSelector,
) -> str:
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
prior_schema_links_str = (
"["
+ ",".join(["""'{}'->{}""".format(k, v) for k, v in prior_schema_links.items()])
+ "]"
)
example_prompt_template = PromptTemplate(input_variables=["table_name", "fields_list", "prior_schema_links", "question", "analysis", "schema_links"],
template="Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schema_links}")
example_prompt_template = PromptTemplate(
input_variables=[
"table_name",
"fields_list",
"prior_schema_links",
"question",
"analysis",
"schema_links",
],
template="Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schema_links}",
)
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links"
@@ -33,28 +47,39 @@ def schema_linking_exampler(user_query: str,
example_separator="\n\n",
prefix=instruction,
input_variables=["table_name", "fields_list", "prior_schema_links", "question"],
suffix=schema_linking_prompt
)
suffix=schema_linking_prompt,
)
schema_linking_example_prompt = schema_linking_example_prompt_template.format(table_name=domain_name,
fields_list=fields_list,
prior_schema_links=prior_schema_links_str,
question=user_query)
schema_linking_example_prompt = schema_linking_example_prompt_template.format(
table_name=domain_name,
fields_list=fields_list,
prior_schema_links=prior_schema_links_str,
question=user_query,
)
return schema_linking_example_prompt
def sql_exampler(user_query: str,
domain_name: str,
schema_link_str: str,
data_date: str,
example_selector: SemanticSimilarityExampleSelector,
) -> str:
def sql_exampler(
user_query: str,
domain_name: str,
schema_link_str: str,
data_date: str,
example_selector: SemanticSimilarityExampleSelector,
) -> str:
instruction = "# 根据schema_links为每个问题生成SQL查询语句"
sql_example_prompt_template = PromptTemplate(input_variables=["question", "current_date", "table_name", "schema_links", "sql"],
template="问题:{question}\nCurrent_date:{current_date}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:{sql}")
sql_example_prompt_template = PromptTemplate(
input_variables=[
"question",
"current_date",
"table_name",
"schema_links",
"sql",
],
template="问题:{question}\nCurrent_date:{current_date}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:{sql}",
)
sql_prompt = "问题:{question}\nCurrent_date:{current_date}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:"
@@ -64,30 +89,51 @@ def sql_exampler(user_query: str,
example_separator="\n\n",
prefix=instruction,
input_variables=["question", "current_date", "table_name", "schema_links"],
suffix=sql_prompt
)
suffix=sql_prompt,
)
sql_example_prompt = sql_example_prompt_template.format(question=user_query,
current_date=data_date,
table_name=domain_name,
schema_links=schema_link_str)
sql_example_prompt = sql_example_prompt_template.format(
question=user_query,
current_date=data_date,
table_name=domain_name,
schema_links=schema_link_str,
)
return sql_example_prompt
def schema_linking_sql_combo_examplar(user_query: str,
domain_name: str,
data_date : str,
fields_list: List[str],
prior_schema_links: Mapping[str,str],
example_selector: SemanticSimilarityExampleSelector) -> str:
def schema_linking_sql_combo_examplar(
user_query: str,
domain_name: str,
data_date: str,
fields_list: List[str],
prior_schema_links: Mapping[str, str],
example_selector: SemanticSimilarityExampleSelector,
) -> str:
prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']'
prior_schema_links_str = (
"["
+ ",".join(["""'{}'->{}""".format(k, v) for k, v in prior_schema_links.items()])
+ "]"
)
example_prompt_template = PromptTemplate(input_variables=["table_name", "fields_list", "prior_schema_links", "current_date", "question", "analysis", "schema_links", "sql"],
template="Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\nCurrent_date:{current_date}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schema_links}\nSQL:{sql}")
example_prompt_template = PromptTemplate(
input_variables=[
"table_name",
"fields_list",
"prior_schema_links",
"current_date",
"question",
"analysis",
"schema_links",
"sql",
],
template="Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\nCurrent_date:{current_date}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schema_links}\nSQL:{sql}",
)
instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links,再根据schema_links为每个问题生成SQL查询语句"
instruction = (
"# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links,再根据schema_links为每个问题生成SQL查询语句"
)
schema_linking_sql_combo_prompt = "Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\nCurrent_date:{current_date}\n问题:{question}\n分析: 让我们一步一步地思考。"
@@ -96,15 +142,23 @@ def schema_linking_sql_combo_examplar(user_query: str,
example_prompt=example_prompt_template,
example_separator="\n\n",
prefix=instruction,
input_variables=["table_name", "fields_list", "prior_schema_links", "current_date", "question"],
suffix=schema_linking_sql_combo_prompt
input_variables=[
"table_name",
"fields_list",
"prior_schema_links",
"current_date",
"question",
],
suffix=schema_linking_sql_combo_prompt,
)
schema_linking_sql_combo_example_prompt = (
schema_linking_sql_combo_example_prompt_template.format(
table_name=domain_name,
fields_list=fields_list,
prior_schema_links=prior_schema_links_str,
current_date=data_date,
question=user_query,
)
schema_linking_sql_combo_example_prompt = schema_linking_sql_combo_example_prompt_template.format(table_name=domain_name,
fields_list=fields_list,
prior_schema_links=prior_schema_links_str,
current_date=data_date,
question=user_query)
)
return schema_linking_sql_combo_example_prompt

237
chat/core/src/main/python/sql/run.py
View File

@@ -7,133 +7,182 @@ import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from sql.prompt_maker import schema_linking_exampler, sql_exampler, schema_linking_sql_combo_examplar
from sql.constructor import sql_examples_vectorstore, sql_example_selector, reload_sql_example_collection
from sql.output_parser import schema_link_parse, combo_schema_link_parse, combo_sql_parse
from sql.prompt_maker import (
schema_linking_exampler,
sql_exampler,
schema_linking_sql_combo_examplar,
)
from sql.constructor import (
sql_examples_vectorstore,
sql_example_selector,
reload_sql_example_collection,
)
from sql.output_parser import (
schema_link_parse,
combo_schema_link_parse,
combo_sql_parse,
)
from util.llm_instance import llm
from run_config import TEXT2DSL_IS_SHORTCUT
class Text2DSLAgent(object):
def __init__(self):
self.schema_linking_exampler = schema_linking_exampler
self.sql_exampler = sql_exampler
def __init__(self):
self.schema_linking_exampler = schema_linking_exampler
self.sql_exampler = sql_exampler
self.schema_linking_sql_combo_exampler = schema_linking_sql_combo_examplar
self.schema_linking_sql_combo_exampler = schema_linking_sql_combo_examplar
self.sql_examples_vectorstore = sql_examples_vectorstore
self.sql_example_selector = sql_example_selector
self.sql_examples_vectorstore = sql_examples_vectorstore
self.sql_example_selector = sql_example_selector
self.schema_link_parse = schema_link_parse
self.combo_schema_link_parse = combo_schema_link_parse
self.combo_sql_parse = combo_sql_parse
self.schema_link_parse = schema_link_parse
self.combo_schema_link_parse = combo_schema_link_parse
self.combo_sql_parse = combo_sql_parse
self.llm = llm
self.llm = llm
self.is_shortcut = TEXT2DSL_IS_SHORTCUT
self.is_shortcut = TEXT2DSL_IS_SHORTCUT
def update_examples(self, sql_examples, example_nums, is_shortcut):
self.sql_examples_vectorstore, self.sql_example_selector = reload_sql_example_collection(self.sql_examples_vectorstore,
sql_examples,
self.sql_example_selector,
example_nums)
self.is_shortcut = is_shortcut
def update_examples(self, sql_examples, example_nums, is_shortcut):
(
self.sql_examples_vectorstore,
self.sql_example_selector,
) = reload_sql_example_collection(
self.sql_examples_vectorstore,
sql_examples,
self.sql_example_selector,
example_nums,
)
self.is_shortcut = is_shortcut
def query2sql(self, query_text: str,
schema : Union[dict, None] = None,
current_date: str = None,
linking: Union[List[Mapping[str, str]], None] = None
):
def query2sql(
self,
query_text: str,
schema: Union[dict, None] = None,
current_date: str = None,
linking: Union[List[Mapping[str, str]], None] = None,
):
print("query_text: ", query_text)
print("schema: ", schema)
print("current_date: ", current_date)
print("prior_schema_links: ", linking)
print("query_text: ", query_text)
print("schema: ", schema)
print("current_date: ", current_date)
print("prior_schema_links: ", linking)
if linking is not None:
prior_schema_links = {item['fieldValue']:item['fieldName'] for item in linking}
else:
prior_schema_links = {}
if linking is not None:
prior_schema_links = {
item["fieldValue"]: item["fieldName"] for item in linking
}
else:
prior_schema_links = {}
model_name = schema['modelName']
fields_list = schema['fieldNameList']
model_name = schema["modelName"]
fields_list = schema["fieldNameList"]
schema_linking_prompt = self.schema_linking_exampler(query_text, model_name, fields_list, prior_schema_links, self.sql_example_selector)
print("schema_linking_prompt->", schema_linking_prompt)
schema_link_output = self.llm(schema_linking_prompt)
schema_link_str = self.schema_link_parse(schema_link_output)
schema_linking_prompt = self.schema_linking_exampler(
query_text,
model_name,
fields_list,
prior_schema_links,
self.sql_example_selector,
)
print("schema_linking_prompt->", schema_linking_prompt)
schema_link_output = self.llm(schema_linking_prompt)
schema_link_str = self.schema_link_parse(schema_link_output)
sql_prompt = self.sql_exampler(query_text, model_name, schema_link_str, current_date, self.sql_example_selector)
print("sql_prompt->", sql_prompt)
sql_output = self.llm(sql_prompt)
sql_prompt = self.sql_exampler(
query_text,
model_name,
schema_link_str,
current_date,
self.sql_example_selector,
)
print("sql_prompt->", sql_prompt)
sql_output = self.llm(sql_prompt)
resp = dict()
resp['query'] = query_text
resp['model'] = model_name
resp['fields'] = fields_list
resp['priorSchemaLinking'] = linking
resp['dataDate'] = current_date
resp = dict()
resp["query"] = query_text
resp["model"] = model_name
resp["fields"] = fields_list
resp["priorSchemaLinking"] = linking
resp["dataDate"] = current_date
resp['analysisOutput'] = schema_link_output
resp['schemaLinkStr'] = schema_link_str
resp["analysisOutput"] = schema_link_output
resp["schemaLinkStr"] = schema_link_str
resp['sqlOutput'] = sql_output
resp["sqlOutput"] = sql_output
print("resp: ", resp)
print("resp: ", resp)
return resp
return resp
def query2sqlcombo(self, query_text: str,
schema : Union[dict, None] = None,
current_date: str = None,
linking: Union[List[Mapping[str, str]], None] = None
):
def query2sqlcombo(
self,
query_text: str,
schema: Union[dict, None] = None,
current_date: str = None,
linking: Union[List[Mapping[str, str]], None] = None,
):
print("query_text: ", query_text)
print("schema: ", schema)
print("current_date: ", current_date)
print("prior_schema_links: ", linking)
print("query_text: ", query_text)
print("schema: ", schema)
print("current_date: ", current_date)
print("prior_schema_links: ", linking)
if linking is not None:
prior_schema_links = {item['fieldValue']:item['fieldName'] for item in linking}
else:
prior_schema_links = {}
if linking is not None:
prior_schema_links = {
item["fieldValue"]: item["fieldName"] for item in linking
}
else:
prior_schema_links = {}
model_name = schema['modelName']
fields_list = schema['fieldNameList']
model_name = schema["modelName"]
fields_list = schema["fieldNameList"]
schema_linking_sql_combo_prompt = self.schema_linking_sql_combo_exampler(query_text, model_name, current_date, fields_list,
prior_schema_links, self.sql_example_selector)
print("schema_linking_sql_combo_prompt->", schema_linking_sql_combo_prompt)
schema_linking_sql_combo_output = self.llm(schema_linking_sql_combo_prompt)
schema_linking_sql_combo_prompt = self.schema_linking_sql_combo_exampler(
query_text,
model_name,
current_date,
fields_list,
prior_schema_links,
self.sql_example_selector,
)
print("schema_linking_sql_combo_prompt->", schema_linking_sql_combo_prompt)
schema_linking_sql_combo_output = self.llm(schema_linking_sql_combo_prompt)
schema_linking_str = self.combo_schema_link_parse(schema_linking_sql_combo_output)
sql_str = self.combo_sql_parse(schema_linking_sql_combo_output)
schema_linking_str = self.combo_schema_link_parse(
schema_linking_sql_combo_output
)
sql_str = self.combo_sql_parse(schema_linking_sql_combo_output)
resp = dict()
resp['query'] = query_text
resp['model'] = model_name
resp['fields'] = fields_list
resp['priorSchemaLinking'] = prior_schema_links
resp['dataDate'] = current_date
resp = dict()
resp["query"] = query_text
resp["model"] = model_name
resp["fields"] = fields_list
resp["priorSchemaLinking"] = prior_schema_links
resp["dataDate"] = current_date
resp['analysisOutput'] = schema_linking_sql_combo_output
resp['schemaLinkStr'] = schema_linking_str
resp['sqlOutput'] = sql_str
resp["analysisOutput"] = schema_linking_sql_combo_output
resp["schemaLinkStr"] = schema_linking_str
resp["sqlOutput"] = sql_str
print("resp: ", resp)
print("resp: ", resp)
return resp
return resp
def query2sql_run(self, query_text: str,
schema : Union[dict, None] = None,
current_date: str = None,
linking: Union[List[Mapping[str, str]], None] = None):
def query2sql_run(
self,
query_text: str,
schema: Union[dict, None] = None,
current_date: str = None,
linking: Union[List[Mapping[str, str]], None] = None,
):
if self.is_shortcut:
return self.query2sqlcombo(query_text, schema, current_date, linking)
else:
return self.query2sql(query_text, schema, current_date, linking)
if self.is_shortcut:
return self.query2sqlcombo(query_text, schema, current_date, linking)
else:
return self.query2sql(query_text, schema, current_date, linking)
text2sql_agent = Text2DSLAgent()

116
chat/core/src/main/python/supersonic_llmparser.py
View File

@@ -13,11 +13,19 @@ from fastapi import FastAPI, HTTPException
from sql.run import text2sql_agent
from preset_retrieval.run import preset_query_retrieval_run, collection as preset_query_collection
from preset_retrieval.preset_query_db import (add2preset_query_collection, update_preset_query_collection,
empty_preset_query_collection, delete_preset_query_by_ids,
update_preset_query_collection, get_preset_query_by_ids,
preset_query_collection_size)
from preset_retrieval.run import (
preset_query_retrieval_run,
collection as preset_query_collection,
)
from preset_retrieval.preset_query_db import (
add2preset_query_collection,
update_preset_query_collection,
empty_preset_query_collection,
delete_preset_query_by_ids,
update_preset_query_collection,
get_preset_query_by_ids,
preset_query_collection_size,
)
from plugin_call.run import plugin_selection_run
@@ -27,62 +35,64 @@ from run_config import LLMPARSER_PORT
app = FastAPI()
@app.post("/query2sql/")
async def din_query2sql(query_body: Mapping[str, Any]):
if 'queryText' not in query_body:
raise HTTPException(status_code=400,
detail="query_text is not in query_body")
if "queryText" not in query_body:
raise HTTPException(status_code=400, detail="query_text is not in query_body")
else:
query_text = query_body['queryText']
query_text = query_body["queryText"]
if 'schema' not in query_body:
if "schema" not in query_body:
raise HTTPException(status_code=400, detail="schema is not in query_body")
else:
schema = query_body['schema']
schema = query_body["schema"]
if 'currentDate' not in query_body:
if "currentDate" not in query_body:
raise HTTPException(status_code=400, detail="currentDate is not in query_body")
else:
current_date = query_body['currentDate']
current_date = query_body["currentDate"]
if 'linking' not in query_body:
if "linking" not in query_body:
linking = None
else:
linking = query_body['linking']
linking = query_body["linking"]
resp = text2sql_agent.query2sql_run(query_text=query_text,
schema=schema, current_date=current_date, linking=linking)
resp = text2sql_agent.query2sql_run(
query_text=query_text, schema=schema, current_date=current_date, linking=linking
)
return resp
@app.post("/query2sql_setting_update/")
async def query2sql_setting_update(query_body: Mapping[str, Any]):
if 'sqlExamplars' not in query_body:
raise HTTPException(status_code=400,
detail="sqlExamplars is not in query_body")
if "sqlExamplars" not in query_body:
raise HTTPException(status_code=400, detail="sqlExamplars is not in query_body")
else:
sql_examplars = query_body['sqlExamplars']
sql_examplars = query_body["sqlExamplars"]
if 'exampleNums' not in query_body:
if "exampleNums" not in query_body:
raise HTTPException(status_code=400, detail="exampleNums is not in query_body")
else:
example_nums = query_body['exampleNums']
example_nums = query_body["exampleNums"]
if 'isShortcut' not in query_body:
if "isShortcut" not in query_body:
raise HTTPException(status_code=400, detail="isShortcut is not in query_body")
else:
is_shortcut = query_body['isShortcut']
is_shortcut = query_body["isShortcut"]
text2sql_agent.update_examples(sql_examples=sql_examplars, example_nums=example_nums, is_shortcut=is_shortcut)
text2sql_agent.update_examples(
sql_examples=sql_examplars, example_nums=example_nums, is_shortcut=is_shortcut
)
return "success"
@app.post("/preset_query_retrival/")
async def preset_query_retrival(query_text_list: List[str], n_results: int = 5):
parsed_retrieval_res_format = preset_query_retrieval_run(preset_query_collection, query_text_list, n_results)
parsed_retrieval_res_format = preset_query_retrieval_run(
preset_query_collection, query_text_list, n_results
)
return parsed_retrieval_res_format
@@ -93,27 +103,32 @@ async def preset_query_add(preset_info_list: List[Mapping[str, str]]):
preset_query_ids = []
for preset_info in preset_info_list:
preset_queries.append(preset_info['preset_query'])
preset_query_ids.append(preset_info['preset_query_id'])
preset_queries.append(preset_info["preset_query"])
preset_query_ids.append(preset_info["preset_query_id"])
add2preset_query_collection(collection=preset_query_collection,
preset_queries=preset_queries,
preset_query_ids=preset_query_ids)
add2preset_query_collection(
collection=preset_query_collection,
preset_queries=preset_queries,
preset_query_ids=preset_query_ids,
)
return "success"
@app.post("/preset_query_update/")
async def preset_query_update(preset_info_list: List[Mapping[str, str]]):
preset_queries = []
preset_query_ids = []
for preset_info in preset_info_list:
preset_queries.append(preset_info['preset_query'])
preset_query_ids.append(preset_info['preset_query_id'])
preset_queries.append(preset_info["preset_query"])
preset_query_ids.append(preset_info["preset_query_id"])
update_preset_query_collection(collection=preset_query_collection,
preset_queries=preset_queries,
preset_query_ids=preset_query_ids)
update_preset_query_collection(
collection=preset_query_collection,
preset_queries=preset_queries,
preset_query_ids=preset_query_ids,
)
return "success"
@@ -124,39 +139,50 @@ async def preset_query_empty():
return "success"
@app.post("/preset_delete_by_ids/")
async def preset_delete_by_ids(preset_query_ids: List[str]):
delete_preset_query_by_ids(collection=preset_query_collection, preset_query_ids=preset_query_ids)
delete_preset_query_by_ids(
collection=preset_query_collection, preset_query_ids=preset_query_ids
)
return "success"
@app.post("/preset_get_by_ids/")
async def preset_get_by_ids(preset_query_ids: List[str]):
preset_queries = get_preset_query_by_ids(collection=preset_query_collection, preset_query_ids=preset_query_ids)
preset_queries = get_preset_query_by_ids(
collection=preset_query_collection, preset_query_ids=preset_query_ids
)
return preset_queries
@app.get("/preset_query_size/")
async def preset_query_size():
size = preset_query_collection_size(collection=preset_query_collection)
return size
@app.post("/plugin_selection/")
async def tool_selection(query_body: Mapping[str, Any]):
if 'queryText' not in query_body:
if "queryText" not in query_body:
raise HTTPException(status_code=400, detail="query_text is not in query_body")
else:
query_text = query_body['queryText']
query_text = query_body["queryText"]
if 'pluginConfigs' not in query_body:
raise HTTPException(status_code=400, detail="pluginConfigs is not in query_body")
if "pluginConfigs" not in query_body:
raise HTTPException(
status_code=400, detail="pluginConfigs is not in query_body"
)
else:
plugin_configs = query_body['pluginConfigs']
plugin_configs = query_body["pluginConfigs"]
resp = plugin_selection_run(query_text=query_text, plugin_configs=plugin_configs)
return resp
if __name__ == "__main__":
uvicorn.run(app, host=LLMPARSER_HOST, port=LLMPARSER_PORT)

25
chat/core/src/main/python/util/chromadb_instance.py
View File

@@ -7,13 +7,15 @@ from chromadb.config import Settings
from run_config import CHROMA_DB_PERSIST_PATH
client = chromadb.Client(Settings(
chroma_db_impl="duckdb+parquet",
persist_directory=CHROMA_DB_PERSIST_PATH # Optional, defaults to .chromadb/ in the current directory
))
client = chromadb.Client(
Settings(
chroma_db_impl="duckdb+parquet",
persist_directory=CHROMA_DB_PERSIST_PATH, # Optional, defaults to .chromadb/ in the current directory
)
)
def empty_chroma_collection_2(collection:Collection):
def empty_chroma_collection_2(collection: Collection):
collection_name = collection.name
client = collection._client
metadata = collection.metadata
@@ -21,17 +23,18 @@ def empty_chroma_collection_2(collection:Collection):
client.delete_collection(collection_name)
new_collection = client.get_or_create_collection(name=collection_name,
metadata=metadata,
embedding_function=embedding_function)
new_collection = client.get_or_create_collection(
name=collection_name, metadata=metadata, embedding_function=embedding_function
)
size_of_new_collection = new_collection.count()
print(f'Collection {collection_name} emptied. Size of new collection: {size_of_new_collection}')
print(
f"Collection {collection_name} emptied. Size of new collection: {size_of_new_collection}"
)
return new_collection
def empty_chroma_collection(collection:Collection):
def empty_chroma_collection(collection: Collection):
collection.delete()

5
chat/core/src/main/python/util/llm_instance.py
View File

@@ -4,5 +4,6 @@ from langchain.llms import OpenAI
from run_config import MODEL_NAME, OPENAI_API_KEY, TEMPERATURE
llm = OpenAI(openai_api_key=OPENAI_API_KEY, model_name=MODEL_NAME,
temperature=TEMPERATURE)
llm = OpenAI(
openai_api_key=OPENAI_API_KEY, model_name=MODEL_NAME, temperature=TEMPERATURE
)

10
chat/core/src/main/python/util/text2vec.py
View File

@@ -9,6 +9,7 @@ from run_config import HF_TEXT2VEC_MODEL_NAME
hg_embedding = HuggingFaceEmbeddings(model_name=HF_TEXT2VEC_MODEL_NAME)
class Text2VecEmbeddingFunction(EmbeddingFunction):
def __call__(self, texts: Documents) -> Embeddings:
@@ -16,13 +17,8 @@ class Text2VecEmbeddingFunction(EmbeddingFunction):
return embeddings
def get_embeddings(documents:List[str]) -> List[List[float]]:
def get_embeddings(documents: List[str]) -> List[List[float]]:
embeddings = hg_embedding.embed_documents(documents)
return embeddings

8
chat/core/src/main/resources/mapper/ChatQueryDOMapper.xml
View File

@@ -3,7 +3,7 @@
<mapper namespace="com.tencent.supersonic.chat.persistence.mapper.ChatQueryDOMapper">
<resultMap id="BaseResultMap" type="com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO">
<id column="question_id" jdbcType="BIGINT" property="questionId" />
<result column="agent_id" jdbcType="BIGINT" property="agentId" />
<result column="agent_id" jdbcType="INTEGER" property="agentId" />
<result column="create_time" jdbcType="TIMESTAMP" property="createTime" />
<result column="user_name" jdbcType="VARCHAR" property="userName" />
<result column="query_state" jdbcType="INTEGER" property="queryState" />
@@ -77,7 +77,7 @@
query_state, chat_id, score,
feedback, query_text, query_result
)
values (#{questionId,jdbcType=BIGINT}, #{agentId,jdbcType=BIGINT}, #{createTime,jdbcType=TIMESTAMP}, #{userName,jdbcType=VARCHAR},
values (#{questionId,jdbcType=BIGINT}, #{agentId,jdbcType=INTEGER}, #{createTime,jdbcType=TIMESTAMP}, #{userName,jdbcType=VARCHAR},
#{queryState,jdbcType=INTEGER}, #{chatId,jdbcType=BIGINT}, #{score,jdbcType=INTEGER},
#{feedback,jdbcType=VARCHAR}, #{queryText,jdbcType=LONGVARCHAR}, #{queryResult,jdbcType=LONGVARCHAR}
)
@@ -98,9 +98,6 @@
<if test="chatId != null">
chat_id = #{chatId,jdbcType=BIGINT},
</if>
<if test="agentId != null">
agent_id = #{agentId,jdbcType=INTEGER},
</if>
<if test="score != null">
score = #{score,jdbcType=INTEGER},
</if>
@@ -116,5 +113,4 @@
</set>
where question_id = #{questionId,jdbcType=BIGINT}
</update>
</mapper>

2
chat/core/src/main/resources/mapper/custom/ShowCaseCustomMapper.xml
View File

@@ -59,7 +59,7 @@
join (
select distinct chat_id
from s2_chat_query
where query_state = 0 and agent_id = ${agentId}
where query_state = 1 and agent_id = ${agentId}
order by chat_id desc
limit #{start}, #{limit}
) q2

45
chat/core/src/test/java/com/tencent/supersonic/chat/corrector/DateFieldCorrectorTest.java
View File

@@ -1,45 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import static org.mockito.ArgumentMatchers.any;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
class DateFieldCorrectorTest {
@Test
void corrector() {
MockedStatic<DSLDateHelper> dslDateHelper = Mockito.mockStatic(DSLDateHelper.class);
dslDateHelper.when(() -> DSLDateHelper.getReferenceDate(any())).thenReturn("2023-08-14");
DateFieldCorrector dateFieldCorrector = new DateFieldCorrector();
SemanticParseInfo parseInfo = new SemanticParseInfo();
SchemaElement model = new SchemaElement();
model.setId(2L);
parseInfo.setModel(model);
SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder()
.sql("select count(歌曲名) from 歌曲库 ")
.parseInfo(parseInfo)
.build();
dateFieldCorrector.correct(semanticCorrectInfo);
Assert.assertEquals("SELECT count(歌曲名) FROM 歌曲库 WHERE 数据日期 = '2023-08-14'", semanticCorrectInfo.getSql());
semanticCorrectInfo = SemanticCorrectInfo.builder()
.sql("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'")
.parseInfo(parseInfo)
.build();
dateFieldCorrector.correct(semanticCorrectInfo);
Assert.assertEquals("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'", semanticCorrectInfo.getSql());
}
}

65
chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldNameCorrectorTest.java
View File

@@ -1,65 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
class FieldNameCorrectorTest {
@Test
void corrector() {
FieldNameCorrector corrector = new FieldNameCorrector();
SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder()
.sql("select 歌曲名 from 歌曲库 where 专辑照片 = '七里香' and 专辑名 = '流行' and 数据日期 = '2023-08-19'")
.build();
SemanticParseInfo parseInfo = new SemanticParseInfo();
DSLParseResult dslParseResult = new DSLParseResult();
LLMReq llmReq = new LLMReq();
List<ElementValue> linking = new ArrayList<>();
ElementValue elementValue = new ElementValue();
elementValue.setFieldValue("流行");
elementValue.setFieldName("歌曲风格");
linking.add(elementValue);
ElementValue elementValue2 = new ElementValue();
elementValue2.setFieldValue("七里香");
elementValue2.setFieldName("歌曲名");
linking.add(elementValue2);
ElementValue elementValue3 = new ElementValue();
elementValue3.setFieldValue("周杰伦");
elementValue3.setFieldName("歌手名");
linking.add(elementValue3);
ElementValue elementValue4 = new ElementValue();
elementValue4.setFieldValue("流行");
elementValue4.setFieldName("歌曲流派");
linking.add(elementValue4);
llmReq.setLinking(linking);
dslParseResult.setLlmReq(llmReq);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, dslParseResult);
parseInfo.setProperties(properties);
semanticCorrectInfo.setParseInfo(parseInfo);
corrector.correct(semanticCorrectInfo);
Assert.assertEquals("SELECT 歌曲名 FROM 歌曲库 WHERE 歌曲名 = '七里香' AND 歌曲流派 = '流行' AND 数据日期 = '2023-08-19'",
semanticCorrectInfo.getSql());
}
}

71
chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldValueCorrectorTest.java
View File

@@ -1,71 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import static org.mockito.Mockito.when;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.service.SchemaService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
class FieldValueCorrectorTest {
@Test
void corrector() {
MockedStatic<ContextUtils> mockContextUtils = Mockito.mockStatic(ContextUtils.class);
SchemaService mockSchemaService = Mockito.mock(SchemaService.class);
SemanticSchema mockSemanticSchema = Mockito.mock(SemanticSchema.class);
List<SchemaElement> dimensions = new ArrayList<>();
List<SchemaValueMap> schemaValueMaps = new ArrayList<>();
SchemaValueMap value1 = new SchemaValueMap();
value1.setBizName("杰伦");
value1.setTechName("周杰伦");
value1.setAlias(Arrays.asList("周杰倫", "Jay Chou", "周董", "周先生"));
schemaValueMaps.add(value1);
SchemaElement schemaElement = SchemaElement.builder()
.bizName("singer_name")
.name("歌手名")
.model(2L)
.schemaValueMaps(schemaValueMaps)
.build();
dimensions.add(schemaElement);
when(mockSemanticSchema.getDimensions()).thenReturn(dimensions);
when(mockSchemaService.getSemanticSchema()).thenReturn(mockSemanticSchema);
mockContextUtils.when(() -> ContextUtils.getBean(SchemaService.class)).thenReturn(mockSchemaService);
SemanticParseInfo parseInfo = new SemanticParseInfo();
SchemaElement model = new SchemaElement();
model.setId(2L);
parseInfo.setModel(model);
SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder()
.sql("select count(song_name) from 歌曲库 where singer_name = '周先生'")
.parseInfo(parseInfo)
.build();
FieldValueCorrector corrector = new FieldValueCorrector();
corrector.correct(semanticCorrectInfo);
Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", semanticCorrectInfo.getSql());
semanticCorrectInfo.setSql("select count(song_name) from 歌曲库 where singer_name = '杰伦'");
corrector.correct(semanticCorrectInfo);
Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", semanticCorrectInfo.getSql());
}
}

46
chat/core/src/test/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrectorTest.java
View File

@@ -1,46 +0,0 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
class SelectFieldAppendCorrectorTest {
@Test
void corrector() {
SelectFieldAppendCorrector corrector = new SelectFieldAppendCorrector();
SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder()
.sql("select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' "
+ "and sys_imp_date = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11")
.build();
corrector.correct(semanticCorrectInfo);
Assert.assertEquals(
"SELECT 歌曲名, 歌手名, 播放量, 歌曲发布时, 发布日期 FROM 歌曲库 WHERE "
+ "datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '邓紫棋' "
+ "AND sys_imp_date = '2023-08-09' AND 歌曲发布时 = '2023-08-01'"
+ " ORDER BY 播放量 DESC LIMIT 11", semanticCorrectInfo.getSql());
semanticCorrectInfo.setSql("select 用户名 from 内容库产品 where datediff('day', 数据日期, '2023-09-14') <= 30"
+ " group by 用户名 having sum(访问次数) > 2000");
corrector.correct(semanticCorrectInfo);
Assert.assertEquals(
"SELECT 用户名, sum(访问次数) FROM 内容库产品 WHERE "
+ "datediff('day', 数据日期, '2023-09-14') <= 30 "
+ "GROUP BY 用户名 HAVING sum(访问次数) > 2000", semanticCorrectInfo.getSql());
semanticCorrectInfo.setSql("SELECT 用户名, sum(访问次数) FROM 内容库产品 WHERE "
+ "datediff('day', 数据日期, '2023-09-14') <= 30 "
+ "GROUP BY 用户名 HAVING sum(访问次数) > 2000");
corrector.correct(semanticCorrectInfo);
Assert.assertEquals(
"SELECT 用户名, sum(访问次数) FROM 内容库产品 WHERE "
+ "datediff('day', 数据日期, '2023-09-14') <= 30 "
+ "GROUP BY 用户名 HAVING sum(访问次数) > 2000", semanticCorrectInfo.getSql());
}
}

9
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java
View File

@@ -7,8 +7,8 @@ import com.tencent.supersonic.knowledge.service.WordService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationStartedEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@@ -17,7 +17,8 @@ import java.util.concurrent.CompletableFuture;
@Slf4j
@Component
public class ApplicationStartedListener implements ApplicationListener<ApplicationStartedEvent> {
@Order(5)
public class ApplicationStartedListener implements CommandLineRunner {
@Autowired
private KnowledgeService knowledgeService;
@@ -27,7 +28,7 @@ public class ApplicationStartedListener implements ApplicationListener<Applicati
private SchemaService schemaService;
@Override
public void onApplicationEvent(ApplicationStartedEvent event) {
public void run(String... args) {
updateKnowledgeDimValue();
}

9
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticLayer.java
View File

@@ -4,18 +4,13 @@ import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.common.pojo.ResultData;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -24,10 +19,6 @@ public abstract class BaseSemanticLayer implements SemanticLayer {
protected final Cache<String, List<ModelSchemaResp>> modelSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build();
protected ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>> structTypeRef =
new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
};
@SneakyThrows
public List<ModelSchemaResp> fetchModelSchema(List<Long> ids, Boolean cacheEnable) {
if (cacheEnable) {

14
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java
View File

@@ -57,17 +57,13 @@ public class LocalSemanticLayer extends BaseSemanticLayer {
}
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) {
try {
queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryDslReq, user);
QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object),
queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryDslReq, user);
QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object),
QueryResultWithSchemaResp.class);
return queryResultWithSchemaResp;
} catch (Exception e) {
log.info("queryByDsl has an exception:{}", e);
}
return null;
return queryResultWithSchemaResp;
}
@Override

2
common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java
View File

@@ -10,7 +10,7 @@ public enum AggOperatorEnum {
SUM("SUM"),
DISTINCT("DISTINCT"),
COUNT_DISTINCT("COUNT_DISTINCT"),
TOPN("TOPN"),

32
dev/reformat Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -x
PROFILES="-P "
# python style checks rely on `black` in path
if ! command -v black &> /dev/null
then
echo "Skip Python lint since 'black' is not available. Please install 'black' by running 'pip install black==22.3.0'"
else
PROFILES="${PROFILES} spotless-python"
fi
mvn spotless:apply $PROFILES

BIN
docs/images/agent.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 14 KiB

BIN
docs/images/chat_config.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 84 KiB

BIN
docs/images/database.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 90 KiB

BIN
docs/images/datasource_base_info.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 220 KiB

BIN
docs/images/datasource_create.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 108 KiB

BIN
docs/images/datasource_extend_info.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 260 KiB

BIN
docs/images/datasource_list.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 102 KiB

BIN
docs/images/datasource_sql.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 173 KiB

BIN
docs/images/detail_default.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 28 KiB

BIN
docs/images/detail_entity.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 133 KiB

BIN
docs/images/dimension_create.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 358 KiB

BIN
docs/images/dimension_list.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 114 KiB

BIN
docs/images/domain.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/images/metric_base_info.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 297 KiB

BIN
docs/images/metric_default.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 18 KiB

BIN
docs/images/metric_list.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 275 KiB

BIN
docs/images/metric_sql_info.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 295 KiB

BIN
docs/images/model.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 9.8 KiB

BIN
docs/images/nlp_config.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 111 KiB

BIN
docs/images/plugin.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 28 KiB

BIN
docs/images/text2sql_config.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 107 KiB

BIN
docs/images/visibility_dim_value.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 69 KiB

BIN
docs/images/visibility_dim_value_show.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 48 KiB

BIN
docs/images/visibility_item.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 68 KiB

BIN
docs/images/wechat_contact.jpeg
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 182 KiB

After

Width:  |  Height:  |  Size: 155 KiB

26
docs/userguides/llm_config_cn.md
View File

@@ -1,26 +0,0 @@
# LLM模型配置
### **简介**
语言模型的使用是超音数的重要一环。能显著增强对用户的问题的理解能力，是通过对话形式与用户交互的基石之一。在本项目中对语言模型能力的应用主要在 LLM 和 Embedding 两方面;默认使用的模型中，LLM选用闭源模型 gpt-3.5-turbo-16k，Embedding模型选用开源模型 GanymedeNil/text2vec-large-chinese。用户可以根据自己实际需求进行配置更改。
### **配置方式**
<div align="left" >
<img src=../images/nlp_config.png width="70%"/>
<p>图1-1 LLM配置文件</p>
</div>
1. LLM模型相关的配置，在 supersonic/chat/core/src/main/python/llm/run_config.py 进行配置。
2. LLM采用OpenAI的闭源模型 gpt-3.5-turbo-16k，在使用时需要提供OpenAI的API-Key才能调用LLM模型，通过 OPENAI_API_KEY 变量进行配置。
3. Embedding模型采用开源模型 GanymedeNil/text2vec-large-chinese，通过 HF_TEXT2VEC_MODEL_NAME 变量进行位置，为了使用方便采用托管在HuggingFace的源，初次启动时自动下载模型文件。
### **FAQ**
1. 可以用开源的LLM模型替代OpenAI的GPT模型吗？
- 暂时不能。我们测试过大部分主流的开源LLM，在实际使用中，在本项目需要LLM提供的逻辑推理和代码生成场景上，开源模型还不能满足需求。
- 我们会持续跟进开源LLM的最新进展，在有满足要求的开源LLM后，在项目中集成私有化部署开源LLM的能力。
2. GPT4、GPT3.5、GPT3.5-16k 这几个模型用哪个比较好？
- GPT3.5、GPT3.5-16k 均能基本满足要求，但会有输出结果不稳定的情况；GPT3.5的token长度限制为4k，在现有CoT策略下，容易出现超过长度限制的情况。
- GPT4的输出更稳定，但费用成本远超GPT3.5，可以根据实际使用场景进行选择。
3. Embedding模型用其他的可以吗？
- 可以。可以以该项目[text2vec]([URL](https://github.com/shibing624/text2vec))的榜单作为参考，然后在HuggingFace找到对应模型的model card，修改HF_TEXT2VEC_MODEL_NAME变量的取值。

29
docs/userguides/text2sql_cn.md
View File

@@ -1,29 +0,0 @@
# text2sql功能相关配置
### **简介**
text2sql的功能实现，高度依赖对LLM的应用。通过LLM生成SQL的过程中，利用小样本(few-shots-examples)通过思维链(chain-of-thoughts)的方式对LLM in-context-learning的能力进行引导，对于生成较为稳定且符合下游语法解析规则的SQL非常重要。用户可以根据自身需要，对样本池及样本的数量进行配置，使其更加符合自身业务特点。
### **配置方式**
1. 样本池的配置。
- supersonic/chat/core/src/main/python/few_shot_example/sql_exampler.py 为样本池配置文件。用户可以以已有的样本作为参考，配置更贴近自身业务需求的样本，用于更好的引导LLM生成SQL。
2. 样本数量的配置。
- 在 supersonic/chat/core/src/main/python/run_config.py 中通过 TEXT2DSL_FEW_SHOTS_EXAMPLE_NUM 变量进行配置。
- 默认值为15，为项目在内部实践后较优的经验值。样本少太少，对导致LLM在生成SQL的过程中缺少引导和示范，生成的SQL会更不稳定；样本太多，会增加生成SQL需要的时间和LLM的token消耗（或超过LLM的token上限）。
3. SQL生成方式的配置
- 在 supersonic/chat/core/src/main/python/run_config.py 中通过 TEXT2DSL_IS_SHORTCUT 变量进行配置。
- 默认值为False；当为False时，会调用2次LLM生成SQL；当为True时，会只调用1次LLM生成SQL。相较于2次LLM调用生成的SQL，耗时会减少30-40%，token的消耗量会减少30%左右，但生成的SQL正确率会有所下降。
<div align="left" >
<img src=../images/text2sql_config.png width="70%"/>
<p>图1-1 配置文件</p>
</div>
### **运行中更新配置的脚本**
1. 如果在启动项目后，用户需要对text2sql功能的相关配置进行调试，可以在修改相关配置文件后，通过以下2种方式让配置在项目运行中让配置生效。
- 执行 supersonic-daemon.sh reload llmparser
- 执行 python examples_reload_run.py
### **FAQ**
1. 生成一个SQL需要消耗的的LLM token数量太多了，按照openAI对token的收费标准，生成一个SQL太贵了，可以少用一些token吗？
- 可以。 用户可以根据自身需求，如配置方式1.中所示，修改样本池中的样本，选用一些更加简短的样本。如配置方式2.中所示，减少使用的样本数量。配置方式3.中所示，只调用1次LLM生成SQL。
- 需要注意，样本和样本数量的选择对生成SQL的质量有很大的影响。过于激进的降低输入的token数量可能会降低生成SQL的质量。需要用户根据自身业务特点实测后进行平衡。

15
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -31,12 +31,9 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.corrector.DateFieldCorrector, \
com.tencent.supersonic.chat.corrector.FunctionAliasCorrector, \
com.tencent.supersonic.chat.corrector.FieldNameCorrector, \
com.tencent.supersonic.chat.corrector.FieldCorrector, \
com.tencent.supersonic.chat.corrector.FunctionCorrector, \
com.tencent.supersonic.chat.corrector.TableNameCorrector, \
com.tencent.supersonic.chat.corrector.QueryFilterAppend, \
com.tencent.supersonic.chat.corrector.SelectFieldAppendCorrector, \
com.tencent.supersonic.chat.corrector.FieldValueCorrector
com.tencent.supersonic.chat.corrector.GlobalCorrector, \
com.tencent.supersonic.chat.corrector.TableCorrector, \
com.tencent.supersonic.chat.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.corrector.SelectCorrector, \
com.tencent.supersonic.chat.corrector.WhereCorrector, \
com.tencent.supersonic.chat.corrector.HavingCorrector

199
launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java Normal file
View File

@@ -0,0 +1,199 @@
package com.tencent.supersonic;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.IdentifyTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.Identify;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.request.DatasourceReq;
import com.tencent.supersonic.semantic.api.model.request.DomainReq;
import com.tencent.supersonic.semantic.api.model.request.ModelReq;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@Component
@Slf4j
@Order(2)
public class LoadBenchMarkDemo implements CommandLineRunner {
private User user = User.getFakeUser();
@Value("${spring.h2.demo.enabled:false}")
private boolean demoEnable;
@Autowired
private DomainService domainService;
@Autowired
private ModelService modelService;
@Autowired
private DatasourceService datasourceService;
@Override
public void run(String... args) {
if (!demoEnable) {
return;
}
try {
addDomain();
addModel_1();
addDatasource_1();
addDatasource_2();
addDatasource_3();
addDatasource_4();
} catch (Exception e) {
log.error("Failed to add bench mark demo data", e);
}
}
public void addDomain() {
DomainReq domainReq = new DomainReq();
domainReq.setName("测评数据-音乐");
domainReq.setBizName("music");
domainReq.setParentId(0L);
domainReq.setViewers(Arrays.asList("admin", "tom", "jack"));
domainReq.setViewOrgs(Collections.singletonList("admin"));
domainReq.setAdmins(Collections.singletonList("admin"));
domainReq.setAdminOrgs(Collections.emptyList());
domainService.createDomain(domainReq, user);
}
public void addModel_1() {
ModelReq modelReq = new ModelReq();
modelReq.setName("测评数据-音乐");
modelReq.setBizName("music");
modelReq.setDomainId(2L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
modelService.createModel(modelReq, user);
}
public void addDatasource_1() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("艺术类型");
datasourceReq.setBizName("genre");
datasourceReq.setDescription("艺术类型");
datasourceReq.setDatabaseId(1L);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
dimensions.add(new Dim("活跃区域", "most_popular_in", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("音乐类型名称", IdentifyTypeEnum.primary.name(), "g_name"));
datasourceReq.setIdentifiers(identifiers);
List<Measure> measures = new ArrayList<>();
Measure measure = new Measure("评分", "rating", AggOperatorEnum.SUM.name(), 0);
measures.add(measure);
datasourceReq.setMeasures(measures);
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre");
datasourceService.createDatasource(datasourceReq, user);
}
public void addDatasource_2() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("艺术家");
datasourceReq.setBizName("artist");
datasourceReq.setDescription("艺术家");
datasourceReq.setDatabaseId(1L);
List<Dim> dimensions = new ArrayList<>();
dimensions.add(new Dim("国籍", "country", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("性别", "gender", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.primary.name(), "artist_name"));
identifiers.add(new Identify("音乐类型名称", IdentifyTypeEnum.foreign.name(), "g_name"));
datasourceReq.setIdentifiers(identifiers);
datasourceReq.setMeasures(Collections.emptyList());
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist");
datasourceService.createDatasource(datasourceReq, user);
}
public void addDatasource_3() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("文件");
datasourceReq.setBizName("files");
datasourceReq.setDescription("文件");
datasourceReq.setDatabaseId(1L);
List<Dim> dimensions = new ArrayList<>();
dimensions.add(new Dim("持续时间", "duration", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("文件格式", "formats", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.primary.name(), "f_id"));
identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name"));
datasourceReq.setIdentifiers(identifiers);
datasourceReq.setMeasures(Collections.emptyList());
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files");
datasourceService.createDatasource(datasourceReq, user);
}
public void addDatasource_4() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("歌曲");
datasourceReq.setBizName("song");
datasourceReq.setDescription("歌曲");
datasourceReq.setDatabaseId(1L);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
dimensions.add(new Dim("国家", "country", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("语种", "languages", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("发行时间", "releasedate", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("歌曲名称", IdentifyTypeEnum.primary.name(), "song_name"));
identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.foreign.name(), "f_id"));
datasourceReq.setIdentifiers(identifiers);
List<Measure> measures = new ArrayList<>();
measures.add(new Measure("分辨率", "resolution", AggOperatorEnum.SUM.name(), 1));
measures.add(new Measure("评分", "rating", AggOperatorEnum.SUM.name(), 1));
datasourceReq.setMeasures(measures);
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, "
+ " rating, languages, releasedate, resolution FROM song");
datasourceService.createDatasource(datasourceReq, user);
}
}

334
launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java Normal file
View File

@@ -0,0 +1,334 @@
package com.tencent.supersonic;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.IdentifyTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.Entity;
import com.tencent.supersonic.semantic.api.model.pojo.Identify;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
import com.tencent.supersonic.semantic.api.model.request.DatabaseReq;
import com.tencent.supersonic.semantic.api.model.request.DatasourceReq;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.DomainReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
import com.tencent.supersonic.semantic.api.model.request.ModelReq;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@Component
@Slf4j
@Order(1)
public class LoadModelDataDemo implements CommandLineRunner {
private User user = User.getFakeUser();
@Value("${spring.h2.demo.enabled:false}")
private boolean demoEnable;
@Autowired
private DatabaseService databaseService;
@Autowired
private DomainService domainService;
@Autowired
private ModelService modelService;
@Autowired
private DatasourceService datasourceService;
@Autowired
private DimensionService dimensionService;
@Autowired
private MetricService metricService;
@Autowired
private AuthService authService;
@Override
public void run(String... args) {
if (!demoEnable) {
return;
}
try {
addDatabase();
addDomain();
addModel_1();
addDatasource_1();
addDatasource_2();
addDatasource_3();
addModel_2();
addDatasource_4();
updateDimension();
updateMetric();
addAuthGroup_1();
addAuthGroup_2();
} catch (Exception e) {
log.error("Failed to add model demo data", e);
}
}
public void addDatabase() {
DatabaseReq databaseReq = new DatabaseReq();
databaseReq.setName("H2数据实例");
databaseReq.setDescription("样例数据库实例");
databaseReq.setType("h2");
databaseReq.setUrl("jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false");
databaseReq.setUsername("root");
databaseReq.setPassword("semantic");
databaseService.createOrUpdateDatabase(databaseReq, user);
}
public void addDomain() {
DomainReq domainReq = new DomainReq();
domainReq.setName("超音数");
domainReq.setBizName("supersonic");
domainReq.setParentId(0L);
domainReq.setViewers(Arrays.asList("admin", "tom", "jack"));
domainReq.setViewOrgs(Collections.singletonList("admin"));
domainReq.setAdmins(Collections.singletonList("admin"));
domainReq.setAdminOrgs(Collections.emptyList());
domainService.createDomain(domainReq, user);
}
public void addModel_1() {
ModelReq modelReq = new ModelReq();
modelReq.setName("超音数");
modelReq.setBizName("supersonic");
modelReq.setDomainId(1L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
modelService.createModel(modelReq, user);
}
public void addDatasource_1() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("用户部门");
datasourceReq.setBizName("user_department");
datasourceReq.setDescription("用户部门");
datasourceReq.setDatabaseId(1L);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));
datasourceReq.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
dimensions.add(new Dim("部门", "department",
DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
datasourceReq.setMeasures(Collections.emptyList());
datasourceReq.setQueryType("table_query");
datasourceReq.setTableQuery("PUBLIC.s2_user_department");
datasourceReq.setModelId(1L);
datasourceService.createDatasource(datasourceReq, user);
}
public void addDatasource_2() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("PVUV统计");
datasourceReq.setBizName("s2_pv_uv_statis");
datasourceReq.setDescription("PVUV统计");
datasourceReq.setDatabaseId(1L);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));
datasourceReq.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0);
dimensions.add(dimension2);
datasourceReq.setDimensions(dimensions);
List<Measure> measures = new ArrayList<>();
Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1);
measures.add(measure1);
Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1);
measures.add(measure2);
datasourceReq.setMeasures(measures);
datasourceReq.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis");
datasourceReq.setQueryType("sql_query");
datasourceReq.setModelId(1L);
datasourceService.createDatasource(datasourceReq, user);
}
public void addDatasource_3() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("停留时长统计");
datasourceReq.setBizName("s2_stay_time_statis");
datasourceReq.setDescription("停留时长统计");
datasourceReq.setDatabaseId(1L);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));
datasourceReq.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
Dim dimension2 = new Dim("页面", "page", DimensionTypeEnum.categorical.name(), 1);
dimensions.add(dimension2);
datasourceReq.setDimensions(dimensions);
List<Measure> measures = new ArrayList<>();
Measure measure1 = new Measure("停留时长", "stay_hours", AggregateTypeEnum.SUM.name(), 1);
measures.add(measure1);
datasourceReq.setMeasures(measures);
datasourceReq.setTableQuery("PUBLIC.s2_stay_time_statis");
datasourceReq.setQueryType("table_query");
datasourceReq.setModelId(1L);
datasourceService.createDatasource(datasourceReq, user);
}
public void addModel_2() {
ModelReq modelReq = new ModelReq();
modelReq.setName("艺人库");
modelReq.setBizName("singer");
modelReq.setDomainId(1L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
modelReq.setEntity(new Entity(7L, Arrays.asList("歌手", "艺人")));
modelService.createModel(modelReq, user);
}
public void addDatasource_4() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("艺人库");
datasourceReq.setBizName("singer");
datasourceReq.setDescription("艺人库");
datasourceReq.setDatabaseId(1L);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("歌手名", IdentifyTypeEnum.primary.name(), "singer_name"));
datasourceReq.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
dimensions.add(new Dim("活跃区域", "act_area",
DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("代表作", "song_name",
DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("风格", "genre",
DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
Measure measure1 = new Measure("播放量", "js_play_cnt", "sum", 1);
Measure measure2 = new Measure("下载量", "down_cnt", "sum", 1);
Measure measure3 = new Measure("收藏量", "favor_cnt", "sum", 1);
datasourceReq.setMeasures(Lists.newArrayList(measure1, measure2, measure3));
datasourceReq.setQueryType("table_query");
datasourceReq.setTableQuery("PUBLIC.singer");
datasourceReq.setModelId(2L);
datasourceService.createDatasource(datasourceReq, user);
}
public void updateDimension() throws Exception {
DimensionReq dimensionReq = new DimensionReq();
dimensionReq.setModelId(1L);
dimensionReq.setType(DimensionTypeEnum.categorical.name());
dimensionReq.setId(3L);
dimensionReq.setName("页面");
dimensionReq.setBizName("page");
dimensionReq.setDatasourceId(3L);
dimensionReq.setAlias("page");
dimensionReq.setSemanticType(SemanticTypeEnum.CATEGORY.name());
dimensionReq.setSensitiveLevel(2);
dimensionReq.setDescription("页面");
dimensionReq.setExpr("page");
dimensionReq.setDimValueMaps(Collections.emptyList());
dimensionService.updateDimension(dimensionReq, user);
}
public void updateMetric() throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(1L);
metricReq.setId(3L);
metricReq.setName("停留时长");
metricReq.setBizName("stay_hours");
metricReq.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
metricReq.setDescription("停留时长");
metricReq.setTags(Collections.singletonList("核心指标"));
metricReq.setAlias("访问时长");
MetricTypeParams metricTypeParams = new MetricTypeParams();
metricTypeParams.setExpr("s2_stay_time_statis_stay_hours");
List<Measure> measures = new ArrayList<>();
Measure measure = new Measure("停留时长",
"s2_stay_time_statis_stay_hours", AggOperatorEnum.SUM.getOperator(), 1);
measure.setDatasourceId(3L);
measures.add(measure);
metricTypeParams.setMeasures(measures);
metricReq.setTypeParams(metricTypeParams);
metricService.updateExprMetric(metricReq, user);
}
public void addAuthGroup_1() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId("1");
authGroupReq.setName("admin-permission");
List<AuthRule> authRules = new ArrayList<>();
AuthRule authRule = new AuthRule();
authRule.setMetrics(Collections.singletonList("stay_hours"));
authRule.setDimensions(Collections.singletonList("page"));
authRules.add(authRule);
authGroupReq.setAuthRules(authRules);
authGroupReq.setAuthorizedUsers(Collections.singletonList("jack"));
authGroupReq.setAuthorizedDepartmentIds(Collections.emptyList());
authService.addOrUpdateAuthGroup(authGroupReq);
}
public void addAuthGroup_2() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId("1");
authGroupReq.setName("tom_sales_permission");
List<AuthRule> authRules = new ArrayList<>();
AuthRule authRule = new AuthRule();
authRule.setMetrics(Collections.singletonList("stay_hours"));
authRule.setDimensions(Collections.singletonList("page"));
authRules.add(authRule);
authGroupReq.setAuthRules(authRules);
authGroupReq.setDimensionFilters(Collections.singletonList("department in ('sales')"));
authGroupReq.setDimensionFilterDescription("部门 in [sales]");
authGroupReq.setAuthorizedUsers(Collections.singletonList("tom"));
authGroupReq.setAuthorizedDepartmentIds(Collections.emptyList());
authService.addOrUpdateAuthGroup(authGroupReq);
}
}

15
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -31,12 +31,9 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.corrector.DateFieldCorrector, \
com.tencent.supersonic.chat.corrector.FunctionAliasCorrector, \
com.tencent.supersonic.chat.corrector.FieldNameCorrector, \
com.tencent.supersonic.chat.corrector.FieldCorrector, \
com.tencent.supersonic.chat.corrector.FunctionCorrector, \
com.tencent.supersonic.chat.corrector.TableNameCorrector, \
com.tencent.supersonic.chat.corrector.QueryFilterAppend, \
com.tencent.supersonic.chat.corrector.SelectFieldAppendCorrector, \
com.tencent.supersonic.chat.corrector.FieldValueCorrector
com.tencent.supersonic.chat.corrector.GlobalCorrector, \
com.tencent.supersonic.chat.corrector.TableCorrector, \
com.tencent.supersonic.chat.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.corrector.SelectCorrector, \
com.tencent.supersonic.chat.corrector.WhereCorrector, \
com.tencent.supersonic.chat.corrector.HavingCorrector

31
launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt Normal file
View File

@@ -0,0 +1,31 @@
孟加拉国 _3_8 9000
锡尔赫特、吉大港、库斯蒂亚 _3_8 9000
加拿大 _3_8 9000
美国 _3_8 9000
tagore _3_9 9000
nazrul _3_9 9000
民间 _3_9 9000
现代 _3_9 9000
蓝调 _3_9 9000
流行 _3_9 9000
孟加拉国 _3_10 9000
印度 _3_10 9000
美国 _3_10 9000
英国 _3_10 9000
男性 _3_11 9000
女性 _3_11 9000
Shrikanta _3_12 9000
Prity _3_12 9000
Farida _3_12 9000
Topu _3_12 9000
Enrique _3_12 9000
Michel _3_12 9000
mp4 _3_14 9000
mp3 _3_14 9000
孟加拉语 _3_16 9000
英文 _3_16 9000
Tumi#长袍#尼罗布 _3_18 9000
舒克诺#帕塔尔#努普尔#帕埃 _3_18 9000
阿米·奥帕尔·霍伊 _3_18 9000
我的爱 _3_18 9000
打败它 _3_18 9000

63
launchers/standalone/src/main/resources/db/data-h2.sql
View File

@@ -5,32 +5,6 @@ insert into s2_user (id, `name`, password, display_name, email) values (3, 'tom'
insert into s2_user (id, `name`, password, display_name, email, is_admin) values (4, 'lucy','123456','lucy','lucy@xx.com', 1);
insert into s2_user (id, `name`, password, display_name, email) values (5, 'alice','123456','alice','alice@xx.com');
-- sample models
insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, viewer, view_org) VALUES(1, '超音数', 'supersonic', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 'admin,tom,jack', 'admin' );
insert into s2_model (id, `name`, biz_name, domain_id, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(1, '超音数', 'supersonic', 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','' );
insert into s2_model (id, `name`, biz_name, domain_id, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(2, '艺人库', 'singer', 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','{"entityId": 7, "names": ["歌手", "艺人"]}' );
insert into s2_database (id, `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by, `admin`) VALUES(1, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(1, 1, '停留时长统计', 's2_stay_time_statis', '停留时长统计', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"page","dateFormat":"yyyy-MM-dd","expr":"page","isCreateDimension":0,"type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[{"agg":"sum","bizName":"s2_stay_time_statis_stay_hours","expr":"stay_hours","isCreateMetric":1,"name":"停留时长"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date, page,user_name,stay_hours FROM s2_stay_time_statis"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(2, 1, 'PVUV统计', 's2_pv_uv_statis', 'PVUV统计', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"page","dateFormat":"yyyy-MM-dd","expr":"page","isCreateDimension":0,"type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","expr":"pv","isCreateMetric":1,"name":"访问次数"},{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","expr":"uv","isCreateMetric":1,"name":"访问人数"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(3, 1, '用户部门', 'user_department', '用户部门', 1, '{"dimensions":[{"bizName":"department","dateFormat":"yyyy-MM-dd","expr":"department","isCreateDimension":1,"name":"部门","type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[],"queryType":"sql_query","sqlQuery":"SELECT user_name,department FROM s2_user_department"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(4, 2, '艺人库', 'singer', '艺人库', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"act_area","dateFormat":"yyyy-MM-dd","expr":"act_area","isCreateDimension":1,"name":"活跃区域","type":"categorical"},{"bizName":"song_name","dateFormat":"yyyy-MM-dd","expr":"song_name","isCreateDimension":1,"name":"代表作","type":"categorical"},{"bizName":"genre","dateFormat":"yyyy-MM-dd","expr":"genre","isCreateDimension":1,"name":"风格","type":"categorical"}],"identifiers":[{"bizName":"singer_name","name":"歌手名","type":"primary"}],"measures":[{"agg":"sum","bizName":"music_down_cnt","expr":"down_cnt","isCreateMetric":1,"name":"下载量"},{"agg":"sum","bizName":"music_js_play_cnt","expr":"js_play_cnt","isCreateMetric":1,"name":"播放量"},{"agg":"sum","bizName":"music_favor_cnt","expr":"favor_cnt","isCreateMetric":1,"name":"收藏量"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date,singer_name,act_area,song_name,genre,js_play_cnt,down_cnt,favor_cnt FROM singer "}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource_rela (id , model_id, `datasource_from`, datasource_to, join_key, created_at, created_by, updated_at, updated_by ) VALUES(1, 1, 1, 2, 'user_name', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource_rela (id , model_id, `datasource_from`, datasource_to, join_key, created_at, created_by, updated_at, updated_by ) VALUES(2, 1, 1, 3, 'user_name', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource_rela (id , model_id, `datasource_from`, datasource_to, join_key, created_at, created_by, updated_at, updated_by ) VALUES(3, 1, 2, 3, 'user_name', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type, dim_value_maps) VALUES(1, 1, 3, '部门', 'department', '部门', 1, 0, 'categorical', NULL, 'department', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY', '[{"alias":["人力资源","人力"],"bizName":"人力资源","techName":"HR"},{"alias":["营销","销售"],"bizName":"营销部门","techName":"sales"}]');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(2, 1, 1, '用户名', 'user_name', '用户名', 1, 0, 'primary', NULL, 'user_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(3, 1, 2, '页面', 'page', '页面', 1, 2, 'categorical', NULL, 'page', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(4, 2, 4, '活跃区域', 'act_area', '活跃区域', 1, 2, 'categorical', NULL, 'act_area', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(5, 2, 4, '代表作', 'song_name', '代表作', 1, 2, 'categorical', NULL, 'song_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(6, 2, 4, '风格', 'genre', '风格', 1, 2, 'categorical', NULL, 'genre', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(7, 2, 4, '歌手名', 'singer_name', '歌手名', 1, 2, 'categorical', NULL, 'singer_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(1, 1, '停留时长', 'stay_hours', '停留时长', 1, 2, 'ATOMIC', '{"expr":"s2_stay_time_statis_stay_hours","measures":[{"agg":"sum","expr":"stay_hours","isCreateMetric":1,"datasourceId":1,"bizName":"s2_stay_time_statis_stay_hours","name":"s2_stay_time_statis_stay_hours"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(2, 1, '访问次数', 'pv', '访问次数', 1, 0, 'ATOMIC', ' {"expr":"s2_pv_uv_statis_pv","measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","datasourceId":2,"expr":"pv","isCreateMetric":1,"name":"s2_pv_uv_statis_pv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(3, 1, '访问人数', 'uv', '访问人数', 1, 0, 'ATOMIC', ' {"expr":"s2_pv_uv_statis_uv","measures":[{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","datasourceId":2,"expr":"uv","isCreateMetric":1,"name":"s2_pv_uv_statis_uv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(4, 2, '播放量', 'js_play_cnt', '播放量', 1, 2, 'ATOMIC', '{"expr":"music_js_play_cnt","measures":[{"agg":"sum","expr":"js_play_cnt","isCreateMetric":1,"datasourceId":4,"bizName":"music_js_play_cnt","name":"music_js_play_cnt"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(5, 2, '下载量', 'down_cnt', '下载量', 1, 0, 'ATOMIC', ' {"expr":"music_down_cnt","measures":[{"agg":"sum","bizName":"music_down_cnt","datasourceId":4,"expr":"down_cnt","isCreateMetric":1,"name":"music_down_cnt"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(6, 2, '收藏量', 'favor_cnt', '收藏量', 1, 0, 'ATOMIC', ' {"expr":"music_favor_cnt","measures":[{"agg":"sum","bizName":"music_favor_cnt","datasourceId":4,"expr":"favor_cnt","isCreateMetric":1,"name":"music_favor_cnt"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_available_date_info(`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (1, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
insert into s2_available_date_info(`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
@@ -38,11 +12,6 @@ values (2, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEA
insert into s2_available_date_info(`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
insert into s2_auth_groups (group_id, config)
values (1, '{"modelId":"1","name":"admin-permission","groupId":1,"authRules":[{"metrics":["stay_hours"],"dimensions":["page"]}],"dimensionFilters":[""],"dimensionFilterDescription":"授权admin 页面和停留时长权限","authorizedUsers":["admin"],"authorizedDepartmentIds":[]}');
insert into s2_auth_groups (group_id, config)
values (2, '{"modelId":"1","name":"tom_sales_permission","groupId":2,"authRules":[{"metrics":["stay_hours"],"dimensions":["page"]}],"dimensionFilters":["department in (''sales'')"],"dimensionFilterDescription":"部门 in [sales]", "authorizedUsers":["tom"],"authorizedDepartmentIds":[]}');
-- sample data
INSERT INTO singer (imp_date,singer_name,act_area, song_name,genre,js_play_cnt,down_cnt,favor_cnt) VALUES (DATEADD('DAY', -1, CURRENT_DATE()), '周杰伦', '港台','青花瓷','国风',1000000,1000000,1000000);
INSERT INTO singer (imp_date,singer_name,act_area, song_name,genre,js_play_cnt,down_cnt,favor_cnt) VALUES (DATEADD('DAY', -5, CURRENT_DATE()), '周杰伦', '港台','青花瓷','国风',1000000,1000000,1000000);
@@ -1108,3 +1077,35 @@ INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (
INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (DATEADD('DAY', -19, CURRENT_DATE()), 'alice', '0.8131712486302015', 'p2');
INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (DATEADD('DAY', -15, CURRENT_DATE()), 'lucy', '0.8124302447925607', 'p4');
INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (DATEADD('DAY', -8, CURRENT_DATE()), 'lucy', '0.039935860913407284', 'p2');
insert into genre(g_name,rating,most_popular_in) VALUES ('tagore',8,'孟加拉国');
insert into genre(g_name,rating,most_popular_in) VALUES ('nazrul',7,'孟加拉国');
insert into genre(g_name,rating,most_popular_in) VALUES ('民间',9,'锡尔赫特、吉大港、库斯蒂亚');
insert into genre(g_name,rating,most_popular_in) VALUES ('现代',8,'孟加拉国');
insert into genre(g_name,rating,most_popular_in) VALUES ('蓝调',7,'加拿大');
insert into genre(g_name,rating,most_popular_in) VALUES ('流行',9,'美国');
insert into artist(artist_name,country,gender,g_name) VALUES ('Shrikanta','印度','男性','tagore');
insert into artist(artist_name,country,gender,g_name) VALUES ('Prity','孟加拉国','女性','nazrul');
insert into artist(artist_name,country,gender,g_name) VALUES ('Farida','孟加拉国','女性','民间');
insert into artist(artist_name,country,gender,g_name) VALUES ('Topu','印度','女性','现代');
insert into artist(artist_name,country,gender,g_name) VALUES ('Enrique','美国','男性','蓝调');
insert into artist(artist_name,country,gender,g_name) VALUES ('Michel','英国','男性','流行');
insert into files(f_id,artist_name,file_size,duration,formats) VALUES (1,'Shrikanta','3.78 MB','3:45','mp4');
insert into files(f_id,artist_name,file_size,duration,formats) VALUES (2,'Prity','4.12 MB','2:56','mp3');
insert into files(f_id,artist_name,file_size,duration,formats) VALUES (3,'Farida','3.69 MB','4:12','mp4');
insert into files(f_id,artist_name,file_size,duration,formats) VALUES (4,'Enrique','4.58 MB','5:23','mp4');
insert into files(f_id,artist_name,file_size,duration,formats) VALUES (5,'Michel','5.10 MB','4:34','mp3');
insert into files(f_id,artist_name,file_size,duration,formats) VALUES (6,'Topu','4.10 MB','4:30','mp4');
insert into song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'Tumi 长袍 尼罗布','Shrikanta','印度',1,'tagore',8,'孟加拉语','28-AUG-2011',1080);
insert into song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'舒克诺 帕塔尔 努普尔 帕埃','Prity','孟加拉国',2,'nazrul',5,'孟加拉语','21-SEP-1997',512);
insert into song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'阿米·奥帕尔·霍伊','Farida','孟加拉国',3,'民间',7,'孟加拉语','7-APR-2001',320);
insert into song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'我的爱','Enrique','美国',4,'蓝调',6,'英文','24-JAN-2007',1080);
insert into song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'打败它','Michel','英国',5,'流行',8,'英文','17-MAR-2002',720);
insert into song(imp_date,song_name,artist_name,country,f_id,g_name,rating,languages,releasedate,resolution) VALUES (DATEADD('DAY', 0, CURRENT_DATE()),'阿杰伊阿卡什','Topu','印度',6,'现代',10,'孟加拉语','27-MAR-2004',320);
-- benchmark

43
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -414,4 +414,47 @@ COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
-- benchmark
CREATE TABLE IF NOT EXISTS `genre` (
`g_name` varchar(20) NOT NULL , -- genre name
`rating` INT ,
`most_popular_in` varchar(50) ,
PRIMARY KEY (`g_name`)
);
COMMENT ON TABLE genre IS 'genre';
CREATE TABLE IF NOT EXISTS `artist` (
`artist_name` varchar(50) NOT NULL , -- genre name
`country` varchar(20) ,
`gender` varchar(20) ,
`g_name` varchar(50)
);
COMMENT ON TABLE artist IS 'artist';
CREATE TABLE IF NOT EXISTS `files` (
`f_id` bigINT NOT NULL,
`artist_name` varchar(50) ,
`file_size` varchar(20) ,
`duration` varchar(20) ,
`formats` varchar(20) ,
PRIMARY KEY (`f_id`)
);
COMMENT ON TABLE files IS 'files';
CREATE TABLE IF NOT EXISTS `song` (
`imp_date` varchar(50) ,
`song_name` varchar(50) ,
`artist_name` varchar(50) ,
`country` varchar(20) ,
`f_id` bigINT ,
`g_name` varchar(20) ,
`rating` INT ,
`languages` varchar(20) ,
`releasedate` varchar(50) ,
`resolution` bigINT NOT NULL
);
COMMENT ON TABLE song IS 'song';
-- benchmark

2
launchers/standalone/src/main/resources/hanlp.properties
View File

@@ -1,2 +1,2 @@
root=.
CustomDictionaryPath=data/dictionary/custom/DimValue_1_1.txt;data/dictionary/custom/DimValue_1_2.txt;data/dictionary/custom/DimValue_1_3.txt;
CustomDictionaryPath=data/dictionary/custom/DimValue_1_1.txt;data/dictionary/custom/DimValue_1_2.txt;data/dictionary/custom/DimValue_1_3.txt;data/dictionary/custom/benchmark_cspider.txt;

10
launchers/standalone/src/test/java/com/tencent/supersonic/benchmark/CSpider.java Normal file
View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.benchmark;
import org.junit.Test;
public class CSpider {
@Test
public void case1(){
}
}

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java
View File

@@ -210,7 +210,7 @@ public class MetricQueryTest extends BaseQueryTest {
ChatConfigEditReqReq extendEditCmd = new ChatConfigEditReqReq();
BeanUtils.copyProperties(chatConfig, extendEditCmd);
// add blacklist
List<Long> blackMetrics = Arrays.asList(3L);
List<Long> blackMetrics = Arrays.asList(2L);
extendEditCmd.getChatAggConfig().getVisibility().setBlackMetricIdList(blackMetrics);
configService.editConfig(extendEditCmd, User.getFakeUser());

31
launchers/standalone/src/test/resources/db/data-h2.sql
View File

@@ -4,32 +4,6 @@ insert into s2_user (id, `name`, password, display_name, email) values (2, 'jack
insert into s2_user (id, `name`, password, display_name, email) values (3, 'tom','123456','tom','tom@xx.com');
insert into s2_user (id, `name`, password, display_name, email) values (4, 'lucy','123456','lucy','lucy@xx.com');
-- sample models
insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, viewer, view_org) VALUES(1, '超音数', 'supersonic', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 'admin,tom,jack', 'admin' );
insert into s2_model (id, `name`, biz_name, domain_id, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(1, '超音数', 'supersonic', 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','' );
insert into s2_model (id, `name`, biz_name, domain_id, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(2, '艺人库', 'singer', 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','{"entityId": 7, "names": ["歌手", "艺人"]}' );
insert into s2_database (id, `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by, `admin`) VALUES(1, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(1, 1, '停留时长统计', 's2_stay_time_statis', '停留时长统计', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"page","dateFormat":"yyyy-MM-dd","expr":"page","isCreateDimension":0,"type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[{"agg":"sum","bizName":"s2_stay_time_statis_stay_hours","expr":"stay_hours","isCreateMetric":1,"name":"停留时长"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date, page,user_name,stay_hours FROM s2_stay_time_statis"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(2, 1, 'PVUV统计', 's2_pv_uv_statis', 'PVUV统计', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"page","dateFormat":"yyyy-MM-dd","expr":"page","isCreateDimension":0,"type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","expr":"pv","isCreateMetric":1,"name":"访问次数"},{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","expr":"uv","isCreateMetric":1,"name":"访问人数"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(3, 1, '用户部门', 'user_department', '用户部门', 1, '{"dimensions":[{"bizName":"department","dateFormat":"yyyy-MM-dd","expr":"department","isCreateDimension":1,"name":"部门","type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[],"queryType":"sql_query","sqlQuery":"SELECT user_name,department FROM s2_user_department"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(4, 2, '艺人库', 'singer', '艺人库', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"act_area","dateFormat":"yyyy-MM-dd","expr":"act_area","isCreateDimension":1,"name":"活跃区域","type":"categorical"},{"bizName":"song_name","dateFormat":"yyyy-MM-dd","expr":"song_name","isCreateDimension":1,"name":"代表作","type":"categorical"},{"bizName":"genre","dateFormat":"yyyy-MM-dd","expr":"genre","isCreateDimension":1,"name":"风格","type":"categorical"}],"identifiers":[{"bizName":"singer_name","name":"歌手名","type":"primary"}],"measures":[{"agg":"sum","bizName":"music_down_cnt","expr":"down_cnt","isCreateMetric":1,"name":"下载量"},{"agg":"sum","bizName":"music_js_play_cnt","expr":"js_play_cnt","isCreateMetric":1,"name":"播放量"},{"agg":"sum","bizName":"music_favor_cnt","expr":"favor_cnt","isCreateMetric":1,"name":"收藏量"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date,singer_name,act_area,song_name,genre,js_play_cnt,down_cnt,favor_cnt FROM singer "}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource_rela (id , model_id, `datasource_from`, datasource_to, join_key, created_at, created_by, updated_at, updated_by ) VALUES(1, 1, 1, 2, 'user_name', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource_rela (id , model_id, `datasource_from`, datasource_to, join_key, created_at, created_by, updated_at, updated_by ) VALUES(2, 1, 1, 3, 'user_name', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource_rela (id , model_id, `datasource_from`, datasource_to, join_key, created_at, created_by, updated_at, updated_by ) VALUES(3, 1, 2, 3, 'user_name', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type, dim_value_maps) VALUES(1, 1, 3, '部门', 'department', '部门', 1, 0, 'categorical', NULL, 'department', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY', '[{"alias":["人力资源","人力"],"bizName":"人力资源","techName":"HR"},{"alias":["营销","销售"],"bizName":"营销部门","techName":"sales"}]');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(2, 1, 1, '用户名', 'user_name', '用户名', 1, 0, 'primary', NULL, 'user_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(3, 1, 2, '页面', 'page', '页面', 1, 2, 'categorical', NULL, 'page', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(4, 2, 4, '活跃区域', 'act_area', '活跃区域', 1, 2, 'categorical', NULL, 'act_area', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(5, 2, 4, '代表作', 'song_name', '代表作', 1, 2, 'categorical', NULL, 'song_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(6, 2, 4, '风格', 'genre', '风格', 1, 2, 'categorical', NULL, 'genre', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_dimension (id , model_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(7, 2, 4, '歌手名', 'singer_name', '歌手名', 1, 2, 'categorical', NULL, 'singer_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY');
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(1, 1, '停留时长', 'stay_hours', '停留时长', 1, 2, 'ATOMIC', '{"expr":"s2_stay_time_statis_stay_hours","measures":[{"agg":"sum","expr":"stay_hours","isCreateMetric":1,"datasourceId":1,"bizName":"s2_stay_time_statis_stay_hours","name":"s2_stay_time_statis_stay_hours"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(2, 1, '访问次数', 'pv', '访问次数', 1, 0, 'ATOMIC', ' {"expr":"s2_pv_uv_statis_pv","measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","datasourceId":2,"expr":"pv","isCreateMetric":1,"name":"s2_pv_uv_statis_pv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(3, 1, '访问人数', 'uv', '访问人数', 1, 0, 'ATOMIC', ' {"expr":"s2_pv_uv_statis_uv","measures":[{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","datasourceId":2,"expr":"uv","isCreateMetric":1,"name":"s2_pv_uv_statis_uv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(4, 2, '播放量', 'js_play_cnt', '播放量', 1, 2, 'ATOMIC', '{"expr":"music_js_play_cnt","measures":[{"agg":"sum","expr":"js_play_cnt","isCreateMetric":1,"datasourceId":4,"bizName":"music_js_play_cnt","name":"music_js_play_cnt"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(5, 2, '下载量', 'down_cnt', '下载量', 1, 0, 'ATOMIC', ' {"expr":"music_down_cnt","measures":[{"agg":"sum","bizName":"music_down_cnt","datasourceId":4,"expr":"down_cnt","isCreateMetric":1,"name":"music_down_cnt"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_metric (id, model_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(6, 2, '收藏量', 'favor_cnt', '收藏量', 1, 0, 'ATOMIC', ' {"expr":"music_favor_cnt","measures":[{"agg":"sum","bizName":"music_favor_cnt","datasourceId":4,"expr":"favor_cnt","isCreateMetric":1,"name":"music_favor_cnt"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL );
insert into s2_available_date_info(`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (1, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
insert into s2_available_date_info(`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
@@ -37,11 +11,6 @@ values (2, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEA
insert into s2_available_date_info(`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
insert into s2_auth_groups (group_id, config)
values (1, '{"modelId":"1","name":"admin-permission","groupId":1,"authRules":[{"metrics":["stay_hours"],"dimensions":["page"]}],"dimensionFilters":[""],"dimensionFilterDescription":"授权admin 页面和停留时长权限","authorizedUsers":["admin"],"authorizedDepartmentIds":[]}');
insert into s2_auth_groups (group_id, config)
values (2, '{"modelId":"1","name":"tom_sales_permission","groupId":2,"authRules":[{"metrics":["stay_hours"],"dimensions":["page"]}],"dimensionFilters":["department in (''sales'')"],"dimensionFilterDescription":"开通 tom sales部门权限", "authorizedUsers":["tom"],"authorizedDepartmentIds":[]}');
-- sample data
INSERT INTO singer (imp_date,singer_name,act_area, song_name,genre,js_play_cnt,down_cnt,favor_cnt) VALUES (DATEADD('DAY', -1, CURRENT_DATE()), '周杰伦', '中国','青花瓷','流行',1000000,1000000,1000000);
INSERT INTO singer (imp_date,singer_name,act_area, song_name,genre,js_play_cnt,down_cnt,favor_cnt) VALUES (DATEADD('DAY', -5, CURRENT_DATE()), '周杰伦', '中国','青花瓷','流行',1000000,1000000,1000000);

43
pom.xml
View File

@@ -65,6 +65,11 @@
<mockito-inline.version>4.5.1</mockito-inline.version>
<jsqlparser.version>4.5</jsqlparser.version>
<revision>0.7.5-SNAPSHOT</revision>
<!-- Do not bump spotless plugin version since 2.30.0 is the latest version supports Java 8-->
<maven.plugin.spotless.version>2.30.0</maven.plugin.spotless.version>
<spotless.python.includes></spotless.python.includes>
<!-- Do not bump black version as decided by spotless maven plugin-->
<spotless.python.black.version>22.3.0</spotless.python.black.version>
</properties>
<dependencyManagement>
@@ -101,6 +106,15 @@
</dependencies>
</dependencyManagement>
<profiles>
<profile>
<id>spotless-python</id>
<properties>
<spotless.python.includes>src/**/*.py</spotless.python.includes>
</properties>
</profile>
</profiles>
<build>
<plugins>
<plugin>
@@ -147,6 +161,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
</plugin>
</plugins>
<pluginManagement>
<plugins>
@@ -185,6 +203,31 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<version>${maven.plugin.spotless.version}</version>
<configuration>
<upToDateChecking>
<enabled>true</enabled>
</upToDateChecking>
<python>
<includes>
<include>${spotless.python.includes}</include>
</includes>
<black>
<version>${spotless.python.black.version}</version>
</black>
</python>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>

9
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/enums/IdentifyTypeEnum.java Normal file
View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.semantic.api.model.enums;
public enum IdentifyTypeEnum {
primary,
foreign,
}

10
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/enums/SemanticTypeEnum.java Normal file
View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.semantic.api.model.enums;
public enum SemanticTypeEnum {
CATEGORY,
ID,
DATE,
NUMBER
}

7
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Dim.java
View File

@@ -25,6 +25,13 @@ public class Dim {
private String bizName;
public Dim(String name, String bizName, String type, Integer isCreateDimension) {
this.name = name;
this.type = type;
this.isCreateDimension = isCreateDimension;
this.bizName = bizName;
}
public static Dim getDefault() {
return new Dim("日期", "time", "2023-05-28",
Constants.DAY_FORMAT,

4
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DimensionTimeTypeParams.java
View File

@@ -10,8 +10,8 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class DimensionTimeTypeParams {
private String isPrimary;
private String isPrimary = "true";
private String timeGranularity;
private String timeGranularity = "day";
}

7
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Measure.java
View File

@@ -28,5 +28,10 @@ public class Measure {
private Long datasourceId;
public Measure(String name, String bizName, String agg, Integer isCreateMetric) {
this.name = name;
this.agg = agg;
this.isCreateMetric = isCreateMetric;
this.bizName = bizName;
}
}

40
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
View File

@@ -1,26 +1,30 @@
package com.tencent.supersonic.semantic.model.application;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component
@@ -96,4 +100,26 @@ public class CatalogImpl implements Catalog {
public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) {
return datasourceService.getItemDate(dimension, metric);
}
@Override
public String getAgg(Long modelId, String metricBizName) {
List<MetricResp> metricResps = getMetrics(modelId);
if (!CollectionUtils.isEmpty(metricResps)) {
Optional<MetricResp> metric = metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(metricBizName)).findFirst();
if (metric.isPresent() && Objects.nonNull(metric.get().getTypeParams()) && !CollectionUtils.isEmpty(
metric.get().getTypeParams().getMeasures())) {
List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(modelId);
if (!CollectionUtils.isEmpty(measureRespList)) {
String measureName = metric.get().getTypeParams().getMeasures().get(0).getBizName();
Optional<MeasureResp> measure = measureRespList.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(measureName)).findFirst();
if (measure.isPresent()) {
return measure.get().getAgg();
}
}
}
}
return "";
}
}

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -221,7 +221,7 @@ public class ModelServiceImpl implements ModelService {
@Override
public Map<Long, String> getModelFullPathMap() {
return getModelList().stream().collect(Collectors.toMap(ModelResp::getId,
return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId,
ModelResp::getFullPath, (k1, k2) -> k1));
}

9
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java
View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.semantic.model.domain;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -16,6 +16,7 @@ import java.util.Set;
public interface Catalog {
DatabaseResp getDatabase(Long id);
DatabaseResp getDatabaseByModelId(Long modelId);
List<DatasourceResp> getDatasourceList(Long modelId);
@@ -36,4 +37,6 @@ public interface Catalog {
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
String getAgg(Long modelId, String metricBizName);
}

9
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/H2Adaptor.java
View File

@@ -29,8 +29,13 @@ public class H2Adaptor extends EngineAdaptor {
@Override
public String getColumnMetaQueryTpl() {
return "SELECT COLUMN_NAME AS name, DATA_TYPE AS dataType\n"
+ "FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA ='%s' AND TABLE_NAME = '%s'";
return "SELECT COLUMN_NAME AS name, "
+ " case DATA_TYPE"
+ " when '12' then 'varchar'"
+ " when '-5' then 'integer'"
+ " when '8' then 'double'"
+ " end AS dataType"
+ " FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA ='%s' AND TABLE_NAME = '%s'";
}
@Override

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java
View File

@@ -10,6 +10,10 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class Identify {
public enum Type {
PRIMARY, FOREIGN
}
private String name;
// primary or foreign

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -25,6 +26,8 @@ public class TableView {
private String alias;
private List<String> primary;
private DataSource dataSource;
public SqlNode build() {
measure.addAll(dimension);
SqlNodeList dimensionNodeList = null;

29
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java
View File

@@ -1,6 +1,11 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;
@@ -9,4 +14,28 @@ public class IdentifyNode extends SemanticNode {
public static SqlNode build(Identify identify, SqlValidatorScope scope) throws Exception {
return parse(identify.getName(), scope);
}
public static Set<String> getIdentifyNames(List<Identify> identifies, Identify.Type type) {
return identifies.stream().filter(i -> type.name().equalsIgnoreCase(i.getType())).map(i -> i.getName())
.collect(Collectors.toSet());
}
public static boolean isForeign(String name, List<Identify> identifies) {
Optional<Identify> identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst();
if (identify.isPresent()) {
return Type.FOREIGN.name().equalsIgnoreCase(identify.get().getType());
}
return false;
}
public static boolean isPrimary(String name, List<Identify> identifies) {
Optional<Identify> identify = identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst();
if (identify.isPresent()) {
return Type.PRIMARY.name().equalsIgnoreCase(identify.get().getType());
}
return false;
}
}

102
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
@@ -12,15 +13,20 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@@ -33,6 +39,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class JoinRender extends Renderer {
@@ -41,6 +48,7 @@ public class JoinRender extends Renderer {
public void render(MetricReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
dataSources = getOrderSource(dataSources);
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
if (queryWhere != null && !queryWhere.isEmpty()) {
@@ -95,6 +103,7 @@ public class JoinRender extends Renderer {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
tableView.setAlias(alias);
tableView.setPrimary(primary);
tableView.setDataSource(dataSource);
if (left == null) {
leftTable = tableView;
left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope));
@@ -246,7 +255,7 @@ public class JoinRender extends Renderer {
private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema,
SqlValidatorScope scope) throws Exception {
log.info(left.getClass().toString());
Set<String> selectLeft = SemanticNode.getSelect(left.getTable());
Set<String> selectRight = SemanticNode.getSelect(right.getTable());
selectLeft.retainAll(selectRight);
@@ -255,6 +264,16 @@ public class JoinRender extends Renderer {
if (!SourceRender.isDimension(on, dataSource, schema)) {
continue;
}
if (IdentifyNode.isForeign(on, left.getDataSource().getIdentifiers())) {
if (!IdentifyNode.isPrimary(on, right.getDataSource().getIdentifiers())) {
continue;
}
}
if (IdentifyNode.isForeign(on, right.getDataSource().getIdentifiers())) {
if (!IdentifyNode.isPrimary(on, left.getDataSource().getIdentifiers())) {
continue;
}
}
List<SqlNode> ons = new ArrayList<>();
ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope));
ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope));
@@ -276,4 +295,85 @@ public class JoinRender extends Renderer {
}
return condition;
}
private List<DataSource> getOrderSource(List<DataSource> dataSources) throws Exception {
if (CollectionUtils.isEmpty(dataSources) || dataSources.size() <= 2) {
return dataSources;
}
Map<String, Set<String>> next = new HashMap<>();
Map<String, Boolean> visited = new HashMap<>();
Map<String, List<Identify>> dataSourceIdentifies = new HashMap<>();
dataSources.stream().forEach(d -> {
next.put(d.getName(), new HashSet<>());
visited.put(d.getName(), false);
dataSourceIdentifies.put(d.getName(), d.getIdentifiers());
});
int cnt = dataSources.size();
List<Map.Entry<String, List<Identify>>> dataSourceIdentifyList = dataSourceIdentifies.entrySet().stream()
.collect(
Collectors.toList());
for (int i = 0; i < cnt; i++) {
for (int j = i + 1; j < cnt; j++) {
Set<String> primaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(),
Type.PRIMARY);
Set<String> foreign = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(),
Type.FOREIGN);
Set<String> nextPrimaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(j).getValue(),
Type.PRIMARY);
Set<String> nextForeign = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(j).getValue(),
Type.FOREIGN);
Set<String> nextAll = new HashSet<>();
nextAll.addAll(nextPrimaries);
nextAll.addAll(nextForeign);
primaries.retainAll(nextPrimaries);
foreign.retainAll(nextPrimaries);
if (primaries.size() > 0 || foreign.size() > 0) {
next.get(dataSourceIdentifyList.get(i).getKey()).add(dataSourceIdentifyList.get(j).getKey());
next.get(dataSourceIdentifyList.get(j).getKey()).add(dataSourceIdentifyList.get(i).getKey());
}
}
}
Queue<String> paths = new ArrayDeque<>();
for (String id : visited.keySet()) {
if (!visited.get(id)) {
joinOrder(cnt, id, next, paths, visited);
if (paths.size() >= cnt) {
break;
}
}
}
if (paths.size() < cnt) {
throw new Exception("datasource cant join,pls check identify :" + dataSources.stream()
.map(d -> d.getName()).collect(
Collectors.joining(",")));
}
List<String> orderList = new ArrayList<>(paths);
Collections.sort(dataSources, new Comparator<DataSource>() {
@Override
public int compare(DataSource o1, DataSource o2) {
return orderList.indexOf(o1.getName()) - orderList.indexOf(o2.getName());
}
});
return dataSources;
}
private static void joinOrder(int cnt, String id, Map<String, Set<String>> next, Queue<String> orders,
Map<String, Boolean> visited) {
visited.put(id, true);
orders.add(id);
if (orders.size() >= cnt) {
return;
}
for (String nextId : next.get(id)) {
if (!visited.get(nextId)) {
joinOrder(cnt, nextId, next, orders, visited);
if (orders.size() >= cnt) {
return;
}
}
}
orders.poll();
visited.put(id, false);
}
}

258
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java Normal file
View File

@@ -0,0 +1,258 @@
package com.tencent.supersonic.semantic.query.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
import java.util.stream.Collectors;
@Service
@Slf4j
public class AuthCommonService {
private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
new SimpleDateFormat(Constants.DAY_FORMAT));
@Autowired
private AuthService authService;
@Autowired
private DimensionService dimensionService;
@Autowired
private MetricService metricService;
@Autowired
private ModelService modelService;
public boolean doModelAdmin(User user, Long modelId) {
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Map<Long, List<ModelResp>> id2modelResp = modelListAdmin.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
return !CollectionUtils.isEmpty(id2modelResp) && id2modelResp.containsKey(modelId);
}
}
public void doModelVisible(User user, Long modelId) {
Boolean visible = true;
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE);
if (CollectionUtils.isEmpty(modelListVisible)) {
visible = false;
} else {
Map<Long, List<ModelResp>> id2domainDesc = modelListVisible.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(modelId)) {
visible = false;
}
}
if (!visible) {
ModelResp modelResp = modelService.getModel(modelId);
String modelName = modelResp.getName();
List<String> admins = modelService.getModelAdmin(modelResp.getId());
String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins);
throw new InvalidPermissionException(message);
}
}
public Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(modelId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(modelId);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(highSensitiveMetrics)) {
highSensitiveMetrics.stream().forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
}
public AuthorizedResourceResp getAuthorizedResource(User user, Long domainId,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId + "");
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
authorizedResource);
return authorizedResource;
}
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId())))
.forEach(col -> resAuthName.add(col.getName()));
}
});
log.info("resAuthName:{}", resAuthName);
return resAuthName;
}
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;
}
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
return false;
}
public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
Long domainId, AuthorizedResourceResp authResource) {
addPromptInfoInfo(domainId, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
public QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
return raw;
}
List<QueryColumn> columns = raw.getColumns();
boolean doDesensitization = false;
for (QueryColumn queryColumn : columns) {
if (need2Apply.contains(queryColumn.getNameEn())) {
doDesensitization = true;
break;
}
}
if (!doDesensitization) {
return raw;
}
QueryResultWithSchemaResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
log.warn("deepCopyResult: ", e);
}
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
return queryResultWithColumns;
}
private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
if (CollectionUtils.isEmpty(need2Apply)) {
return;
}
columns.stream().forEach(col -> {
if (need2Apply.contains(col.getNameEn())) {
col.setAuthorized(false);
}
});
}
private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
log.info("start desensitizationInternal logic");
for (int i = 0; i < result.size(); i++) {
Map<String, Object> row = result.get(i);
Map<String, Object> newRow = new HashMap<>();
for (String col : row.keySet()) {
if (need2Apply.contains(col)) {
newRow.put(col, "****");
} else {
newRow.put(col, row.get(col));
}
}
result.set(i, newRow);
}
}
private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getColumns())) {
String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
});
queryResultWithColumns.setColumns(columns);
}
queryResultWithColumns.setColumns(columns);
List<Map<String, Object>> resultData = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getResultList())) {
for (Map<String, Object> line : raw.getResultList()) {
Map<String, Object> newLine = new HashMap<>();
newLine.putAll(line);
resultData.add(newLine);
}
}
queryResultWithColumns.setResultList(resultData);
return queryResultWithColumns;
}
public void addPromptInfoInfo(Long modelId, QueryResultWithSchemaResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelId);
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
ModelResp modelResp = modelService.getModel(modelId);
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {
descList.add(filter.getDescription());
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -21,6 +21,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
@@ -66,9 +67,16 @@ public class QueryServiceImpl implements QueryService {
}
@Override
public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception {
@DslPermissionAnnotation
@SneakyThrows
public Object queryBySql(QueryDslReq querySqlCmd, User user) {
statUtils.initStatInfo(querySqlCmd, user);
QueryStatement queryStatement = convertToQueryStatement(querySqlCmd, user);
QueryStatement queryStatement = new QueryStatement();
try {
queryStatement = convertToQueryStatement(querySqlCmd, user);
} catch (Exception e) {
log.info("convertToQueryStatement has a exception:{}", e.toString());
}
QueryResultWithSchemaResp results = semanticQueryEngine.execute(queryStatement);
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return results;

Some files were not shown because too many files have changed in this diff Show More