diff --git a/CHANGELOG.md b/CHANGELOG.md index b33b6a7f2..35253a8d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ - All notable changes to this project will be documented in this file. - "Breaking Changes" describes any changes that may break existing functionality or cause compatibility issues with previous versions. + + ## SuperSonic [0.7.4] - 2023-09-10 + + ### Added + - add llm parser config + - add datasource agg_time option + - add function name adaptor in clickhouse + - add dimension and metric show in dsl + - add support for dimensions and metrics to dynamically update the knowledge base + + ### Updated + - update user guide doc + - update query building of plugin in default model + - update some core API constructs to keep naming consistency + - update ConfigureDemo config + - update the association mechanism so that invisible dimensions and metrics will no longer be associated + + ### Fixed + - fix hasAggregateFunction logic in SqlParserSelectHelper ## SuperSonic [0.7.3] - 2023-08-29 diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java index 05d1a8991..b7cd770d2 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java @@ -9,10 +9,12 @@ import org.springframework.context.annotation.Configuration; @Configuration public class AuthenticationConfig { - @Value("${authentication.exclude.path:XXX}") private String excludePath; + @Value("${authentication.include.path:/api}") + private String includePath; + @Value("${authentication.enable:false}") private boolean enabled; @@ -22,5 +24,4 @@ public class AuthenticationConfig { @Value("${authentication.token.http.header.key:Authorization}") private String tokenHttpHeaderKey; - } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/config/TppConfig.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/config/TppConfig.java deleted file mode 100644 index 474de6004..000000000 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/config/TppConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tencent.supersonic.auth.authentication.config; - - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Data -@Configuration -public class TppConfig { - - @Value(value = "${auth.app.secret:}") - private String appSecret; - - @Value(value = "${auth.app.key:}") - private String appKey; - - @Value(value = "${auth.oa.url:}") - private String tppOaUrl; - -} diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java index 0d52c9566..0f24f39c4 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java @@ -42,6 +42,17 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor { return excludePaths.stream().anyMatch(uri::startsWith); } + protected boolean isIncludedUri(String uri) { + String includePathStr = authenticationConfig.getIncludePath(); + if (Strings.isEmpty(includePathStr)) { + return false; + } + List includePaths = Arrays.asList(includePathStr.split(",")); + if (CollectionUtils.isEmpty(includePaths)) { + return false; + } + return includePaths.stream().anyMatch(uri::startsWith); + } protected boolean isInternalRequest(HttpServletRequest request) { String internal = request.getHeader(UserConstants.INTERNAL); diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java index 67daa9547..b1c1c8c80 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java @@ -46,6 +46,10 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor } String uri = request.getServletPath(); + if (!isIncludedUri(uri)) { + return true; + } + if (isExcludedUri(uri)) { return true; } diff --git a/auth/authorization/pom.xml b/auth/authorization/pom.xml index 78907dfd8..96823320f 100644 --- a/auth/authorization/pom.xml +++ b/auth/authorization/pom.xml @@ -35,10 +35,6 @@ com.google.code.gson gson - - com.google.guava - guava - com.tencent.supersonic auth-authentication @@ -48,4 +44,4 @@ - \ No newline at end of file + diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SchemaMapper.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SchemaMapper.java index 79c62f75f..63b47b01c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SchemaMapper.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SchemaMapper.java @@ -3,11 +3,8 @@ package com.tencent.supersonic.chat.api.component; import com.tencent.supersonic.chat.api.pojo.QueryContext; /** - * This interface defines the contract for a schema mapper that identifies references to schema - * elements in natural language queries. - * - * The schema mapper matches queries against the knowledge base which is constructed using the - * schema of semantic models. + * A schema mapper identifies references to schema elements(metrics/dimensions/entities/values) + * in user queries. It matches the query text against the knowledge base. */ public interface SchemaMapper { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticCorrector.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticCorrector.java index e228d1af3..7eda4759d 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticCorrector.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticCorrector.java @@ -1,9 +1,13 @@ package com.tencent.supersonic.chat.api.component; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import net.sf.jsqlparser.JSQLParserException; +/** + * A semantic corrector checks validity of extracted semantic information and + * performs correction and optimization if needed. + */ public interface SemanticCorrector { - CorrectionInfo corrector(CorrectionInfo correctionInfo) throws JSQLParserException; + void correct(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java index fb931fdff..54a9a06ab 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java @@ -18,10 +18,9 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import java.util.List; /** - * This interface defines the contract for a semantic layer that provides a simplified and - * consistent view of data from multiple sources. - * The semantic layer abstracts away the complexity of the underlying data sources and provides - * a unified view of the data that is easier to understand and use. + * A semantic layer provides a simplified and consistent view of data from multiple sources. + * It abstracts away the complexity of the underlying data sources and provides a unified view + * of the data that is easier to understand and use. *

* The interface defines methods for getting metadata as well as querying data in the semantic layer. * Implementations of this interface should provide concrete implementations that interact with the diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticParser.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticParser.java index 198c9ec1b..c928c41a2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticParser.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticParser.java @@ -5,11 +5,9 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; /** - * This interface defines the contract for a semantic parser that can analyze natural language query - * and extract meaning from it. - * - * The semantic parser uses either rule-based or model-based algorithms to identify query intent - * and related semantic items described in the query. + * A semantic parser understands user queries and extracts semantic information. + * It could leverage either rule-based or LLM-based approach to identify query intent + * and extract related semantic items from the query. */ public interface SemanticParser { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java index def007dca..91acf9016 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java @@ -6,8 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import org.apache.calcite.sql.parser.SqlParseException; /** - * This class defines the contract for a semantic query that executes specific type of - * query based on the results of semantic parsing. + * A semantic query executes specific type of query based on the results of semantic parsing. */ public interface SemanticQuery { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java index 99bfd0714..057599569 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java @@ -21,6 +21,10 @@ public class SchemaMapInfo { return modelElementMatches; } + public void setModelElementMatches(Map> modelElementMatches) { + this.modelElementMatches = modelElementMatches; + } + public void setMatchedElements(Long model, List elementMatches) { modelElementMatches.put(model, elementMatches); } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/CorrectionInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticCorrectInfo.java similarity index 92% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/CorrectionInfo.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticCorrectInfo.java index 28caca4bc..1099273dd 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/CorrectionInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticCorrectInfo.java @@ -10,7 +10,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @NoArgsConstructor @Builder -public class CorrectionInfo { +public class SemanticCorrectInfo { private QueryFilters queryFilters; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibility.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibility.java new file mode 100644 index 000000000..d4910839b --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibility.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.chat.api.pojo.request; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class ItemNameVisibility { + + private ItemNameVisibilityInfo aggVisibilityInfo; + + private ItemNameVisibilityInfo detailVisibilityInfo; +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java new file mode 100644 index 000000000..5ca96f41c --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemNameVisibilityInfo.java @@ -0,0 +1,23 @@ +package com.tencent.supersonic.chat.api.pojo.request; + +import lombok.Data; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.List; + + +@Data +@ToString +public class ItemNameVisibilityInfo { + + /** + * invisible dimensions + */ + private List blackDimNameList = new ArrayList<>(); + + /** + * invisible metrics + */ + private List blackMetricNameList = new ArrayList<>(); +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java index c01deee97..b32e55b81 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java @@ -32,6 +32,4 @@ public class KnowledgeInfoReq { * advanced knowledge config for single item */ private KnowledgeAdvancedConfig knowledgeAdvancedConfig; - - } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentTool.java index 126e89245..4ff8efe01 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentTool.java @@ -12,5 +12,4 @@ public class AgentTool { private String name; private AgentToolType type; - } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/aspect/TimeCostAspect.java b/chat/core/src/main/java/com/tencent/supersonic/chat/aspect/TimeCostAspect.java deleted file mode 100644 index 7d4e0de0b..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/aspect/TimeCostAspect.java +++ /dev/null @@ -1,99 +0,0 @@ -/* -//package com.tencent.supersonic.chat.aspect; -// -//import lombok.extern.slf4j.Slf4j; -//import org.aspectj.lang.JoinPoint; -//import org.aspectj.lang.ProceedingJoinPoint; -//import org.aspectj.lang.annotation.*; -//import org.springframework.stereotype.Component; -// -//import java.util.HashMap; -//import java.util.Map; -// -//@Aspect -//@Component -//@Slf4j -//public class TimeCostAspect { -// -// ThreadLocal startTime = new ThreadLocal<>(); -// -// ThreadLocal> map = new ThreadLocal<>(); -// -// @Pointcut("execution(public * com.tencent.supersonic.chat.mapper.HanlpDictMapper.*(*))") -// //@Pointcut("execution(* public com.tencent.supersonic.chat.parser.*.*(..))") -// //@Pointcut("execution(* com.tencent.supersonic.chat.mapper.*Mapper.map(..)) ") -// //@Pointcut("execution(* com.tencent.supersonic.chat.mapper.HanlpDictMapper.map(..)) ") -// //@Pointcut("execution(* com.tencent.supersonic.chat.parser.rule.QueryModeParser.*(..)) ") -// public void point() { -// } -// -// @Around("point()") -// public void doAround(ProceedingJoinPoint joinPoint) throws Throwable { -// long start = System.currentTimeMillis(); -// try { -// log.info("切面开始"); -// Object result = joinPoint.proceed(); -// log.info("切面开始"); -// if (result == null) { -// //如果切到了 没有返回类型的void方法，这里直接返回 -// //return null; -// } -// long end = System.currentTimeMillis(); -// log.info("==================="); -// String targetClassName = joinPoint.getSignature().getDeclaringTypeName(); -// String MethodName = joinPoint.getSignature().getName(); -// String typeStr = joinPoint.getSignature().getDeclaringType().toString().split(" ")[0]; -// log.info("类/接口:" + targetClassName + "(" + typeStr + ")"); -// log.info("方法:" + MethodName); -// Long total = end - start; -// log.info("耗时: " + total + " ms!"); -// map.get().put(targetClassName + "_" + MethodName, total); -// //return result; -// } catch (Throwable e) { -// long end = System.currentTimeMillis(); -// log.info("====around " + joinPoint + "\tUse time : " + (end - start) + " ms with exception : " -// + e.getMessage()); -// throw e; -// } -// } -// -//// //对Controller下面的方法执行前进行切入，初始化开始时间 -//// @Before(value = "execution(* com.appleyk.controller.*.*(..))") -//// public void beforMehhod(JoinPoint jp) { -//// startTime.set(System.currentTimeMillis()); -//// } -//// -//// //对Controller下面的方法执行后进行切入，统计方法执行的次数和耗时情况 -//// //注意，这里的执行方法统计的数据不止包含Controller下面的方法，也包括环绕切入的所有方法的统计信息 -//// @AfterReturning(value = "execution(* com.appleyk.controller.*.*(..))") -//// public void afterMehhod(JoinPoint jp) { -//// long end = System.currentTimeMillis(); -//// long total = end - startTime.get(); -//// String methodName = jp.getSignature().getName(); -//// log.info("连接点方法为：" + methodName + ",执行总耗时为：" +total+"ms"); -//// -//// //重新new一个map -//// Map map = new HashMap<>(); -//////从map2中将最后的 连接点方法给移除了，替换成最终的，避免连接点方法多次进行叠加计算 -//// //由于map2受ThreadLocal的保护，这里不支持remove，因此，需要单开一个map进行数据交接 -//// for(Map.Entry entry:map2.get().entrySet()){ -//// if(entry.getKey().equals(methodName)){ -//// map.put(methodName, total); -//// -//// }else{ -//// map.put(entry.getKey(), entry.getValue()); -//// } -//// } -//// -//// for (Map.Entry entry :map1.get().entrySet()) { -//// for(Map.Entry entry2 :map.entrySet()){ -//// if(entry.getKey().equals(entry2.getKey())){ -//// System.err.println(entry.getKey()+",被调用次数："+entry.getValue()+",综合耗时："+entry2.getValue()+"ms"); -//// } -//// } -//// -//// } -//// } -// -//} -*/ diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java index ec23773bd..2d694044f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java @@ -9,4 +9,7 @@ import org.springframework.context.annotation.Configuration; public class FunctionCallInfoConfig { @Value("${functionCall.url:}") private String url; + + @Value("${funtionCall.plugin.select.path:/plugin_selection}") + private String pluginSelectPath; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/DateFieldCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/DateFieldCorrector.java index 232dfba16..f4221a8d8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/DateFieldCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/DateFieldCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; @@ -12,17 +12,16 @@ import org.springframework.util.CollectionUtils; public class DateFieldCorrector extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { + public void correct(SemanticCorrectInfo semanticCorrectInfo) { - String sql = correctionInfo.getSql(); + String sql = semanticCorrectInfo.getSql(); List whereFields = SqlParserSelectHelper.getWhereFields(sql); if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DATE_FIELD)) { - String currentDate = DSLDateHelper.getCurrentDate(correctionInfo.getParseInfo().getModelId()); + String currentDate = DSLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId()); sql = SqlParserUpdateHelper.addWhere(sql, DATE_FIELD, currentDate); } - correctionInfo.setPreSql(correctionInfo.getSql()); - correctionInfo.setSql(sql); - return correctionInfo; + semanticCorrectInfo.setPreSql(semanticCorrectInfo.getSql()); + semanticCorrectInfo.setSql(sql); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldCorrector.java index d46bb4f28..77cb01c3d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; import lombok.extern.slf4j.Slf4j; @@ -8,12 +8,11 @@ import lombok.extern.slf4j.Slf4j; public class FieldCorrector extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { - String preSql = correctionInfo.getSql(); - correctionInfo.setPreSql(preSql); + public void correct(SemanticCorrectInfo semanticCorrectInfo) { + String preSql = semanticCorrectInfo.getSql(); + semanticCorrectInfo.setPreSql(preSql); String sql = SqlParserUpdateHelper.replaceFields(preSql, - getFieldToBizName(correctionInfo.getParseInfo().getModelId())); - correctionInfo.setSql(sql); - return correctionInfo; + getFieldToBizName(semanticCorrectInfo.getParseInfo().getModelId())); + semanticCorrectInfo.setSql(sql); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldNameCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldNameCorrector.java index b081ebc3f..f94b98253 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldNameCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldNameCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue; @@ -19,32 +19,31 @@ import org.springframework.util.CollectionUtils; public class FieldNameCorrector extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { + public void correct(SemanticCorrectInfo semanticCorrectInfo) { - Object context = correctionInfo.getParseInfo().getProperties().get(Constants.CONTEXT); + Object context = semanticCorrectInfo.getParseInfo().getProperties().get(Constants.CONTEXT); if (Objects.isNull(context)) { - return correctionInfo; + return; } DSLParseResult dslParseResult = JsonUtil.toObject(JsonUtil.toString(context), DSLParseResult.class); if (Objects.isNull(dslParseResult) || Objects.isNull(dslParseResult.getLlmReq())) { - return correctionInfo; + return; } LLMReq llmReq = dslParseResult.getLlmReq(); List linking = llmReq.getLinking(); if (CollectionUtils.isEmpty(linking)) { - return correctionInfo; + return; } Map> fieldValueToFieldNames = linking.stream().collect( Collectors.groupingBy(ElementValue::getFieldValue, Collectors.mapping(ElementValue::getFieldName, Collectors.toSet()))); - String preSql = correctionInfo.getSql(); - correctionInfo.setPreSql(preSql); + String preSql = semanticCorrectInfo.getSql(); + semanticCorrectInfo.setPreSql(preSql); String sql = SqlParserUpdateHelper.replaceFieldNameByValue(preSql, fieldValueToFieldNames); - correctionInfo.setSql(sql); - return correctionInfo; + semanticCorrectInfo.setSql(sql); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldValueCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldValueCorrector.java index bc62c4a80..b660f8946 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldValueCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldValueCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; @@ -20,23 +20,23 @@ import org.springframework.util.CollectionUtils; public class FieldValueCorrector extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { + public void correct(SemanticCorrectInfo semanticCorrectInfo) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - Long modelId = correctionInfo.getParseInfo().getModel().getId(); + Long modelId = semanticCorrectInfo.getParseInfo().getModel().getId(); List dimensions = semanticSchema.getDimensions().stream() .filter(schemaElement -> modelId.equals(schemaElement.getModel())) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dimensions)) { - return correctionInfo; + return; } Map> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions); - String preSql = correctionInfo.getSql(); - correctionInfo.setPreSql(preSql); + String preSql = semanticCorrectInfo.getSql(); + semanticCorrectInfo.setPreSql(preSql); String sql = SqlParserUpdateHelper.replaceValue(preSql, aliasAndBizNameToTechName); - correctionInfo.setSql(sql); - return correctionInfo; + semanticCorrectInfo.setSql(sql); + return; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionCorrector.java index c5af7566c..e0a3a3210 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; import lombok.extern.slf4j.Slf4j; @@ -8,11 +8,10 @@ import lombok.extern.slf4j.Slf4j; public class FunctionCorrector extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { - String preSql = correctionInfo.getSql(); - correctionInfo.setPreSql(preSql); + public void correct(SemanticCorrectInfo semanticCorrectInfo) { + String preSql = semanticCorrectInfo.getSql(); + semanticCorrectInfo.setPreSql(preSql); String sql = SqlParserUpdateHelper.replaceFunction(preSql); - correctionInfo.setSql(sql); - return correctionInfo; + semanticCorrectInfo.setSql(sql); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/QueryFilterAppend.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/QueryFilterAppend.java index 185d5c6a4..4bb63515d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/QueryFilterAppend.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/QueryFilterAppend.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.StringUtil; @@ -18,18 +18,17 @@ import org.apache.commons.lang3.StringUtils; public class QueryFilterAppend extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) throws JSQLParserException { - String queryFilter = getQueryFilter(correctionInfo.getQueryFilters()); - String preSql = correctionInfo.getSql(); + public void correct(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException { + String queryFilter = getQueryFilter(semanticCorrectInfo.getQueryFilters()); + String preSql = semanticCorrectInfo.getSql(); if (StringUtils.isNotEmpty(queryFilter)) { log.info("add queryFilter to preSql :{}", queryFilter); Expression expression = CCJSqlParserUtil.parseCondExpression(queryFilter); String sql = SqlParserUpdateHelper.addWhere(preSql, expression); - correctionInfo.setPreSql(preSql); - correctionInfo.setSql(sql); + semanticCorrectInfo.setPreSql(preSql); + semanticCorrectInfo.setSql(sql); } - return correctionInfo; } private String getQueryFilter(QueryFilters queryFilters) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrector.java index 00a8ceb7a..e906de510 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum; @@ -14,16 +14,16 @@ import org.springframework.util.CollectionUtils; public class SelectFieldAppendCorrector extends BaseSemanticCorrector { @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { - String preSql = correctionInfo.getSql(); + public void correct(SemanticCorrectInfo semanticCorrectInfo) { + String preSql = semanticCorrectInfo.getSql(); if (SqlParserSelectHelper.hasAggregateFunction(preSql)) { - return correctionInfo; + return; } Set selectFields = new HashSet<>(SqlParserSelectHelper.getSelectFields(preSql)); Set whereFields = new HashSet<>(SqlParserSelectHelper.getWhereFields(preSql)); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(whereFields)) { - return correctionInfo; + return; } whereFields.addAll(SqlParserSelectHelper.getOrderByFields(preSql)); @@ -32,8 +32,7 @@ public class SelectFieldAppendCorrector extends BaseSemanticCorrector { whereFields.remove(TimeDimensionEnum.WEEK.getName()); whereFields.remove(TimeDimensionEnum.MONTH.getName()); String replaceFields = SqlParserUpdateHelper.addFieldsToSelect(preSql, new ArrayList<>(whereFields)); - correctionInfo.setPreSql(preSql); - correctionInfo.setSql(replaceFields); - return correctionInfo; + semanticCorrectInfo.setPreSql(preSql); + semanticCorrectInfo.setSql(replaceFields); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableNameCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableNameCorrector.java index df82eeb22..03f9b7ecb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableNameCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableNameCorrector.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; import lombok.extern.slf4j.Slf4j; @@ -10,13 +10,12 @@ public class TableNameCorrector extends BaseSemanticCorrector { public static final String TABLE_PREFIX = "t_"; @Override - public CorrectionInfo corrector(CorrectionInfo correctionInfo) { - Long modelId = correctionInfo.getParseInfo().getModelId(); - String preSql = correctionInfo.getSql(); - correctionInfo.setPreSql(preSql); + public void correct(SemanticCorrectInfo semanticCorrectInfo) { + Long modelId = semanticCorrectInfo.getParseInfo().getModelId(); + String preSql = semanticCorrectInfo.getSql(); + semanticCorrectInfo.setPreSql(preSql); String sql = SqlParserUpdateHelper.replaceTable(preSql, TABLE_PREFIX + modelId); - correctionInfo.setSql(sql); - return correctionInfo; + semanticCorrectInfo.setSql(sql); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java index c4b57d3d3..b007f9ce6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.MapResult; import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder; import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java index b79db9d81..005a57ac5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java @@ -14,6 +14,7 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -49,7 +50,6 @@ public class MapperHelper { } public double getThresholdMatch(List natures) { - log.info("optimizationConfig:{}", optimizationConfig); if (existDimensionValues(natures)) { return optimizationConfig.getDimensionValueThresholdConfig(); } @@ -90,9 +90,20 @@ public class MapperHelper { AgentService agentService = ContextUtils.getBean(AgentService.class); Set detectModelIds = agentService.getDslToolsModelIds(request.getAgentId(), null); + //contains all + if (isContainsAllModel(detectModelIds)) { + if (Objects.nonNull(modelId) && modelId > 0) { + Set result = new HashSet<>(); + result.add(modelId); + return result; + } + return new HashSet<>(); + } + if (Objects.nonNull(detectModelIds)) { detectModelIds = detectModelIds.stream().filter(entry -> entry > 0).collect(Collectors.toSet()); } + if (Objects.nonNull(modelId) && modelId > 0 && Objects.nonNull(detectModelIds)) { if (detectModelIds.contains(modelId)) { Set result = new HashSet<>(); @@ -103,4 +114,8 @@ public class MapperHelper { return detectModelIds; } + private boolean isContainsAllModel(Set detectModelIds) { + return CollectionUtils.isNotEmpty(detectModelIds) && detectModelIds.contains(-1L); + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/SearchMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/SearchMatchStrategy.java index a3299a22b..5a0fdfba0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/SearchMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/SearchMatchStrategy.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.mapper; import com.google.common.collect.Lists; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.MapResult; import com.tencent.supersonic.knowledge.service.SearchService; import java.util.List; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java index 55a4026c9..4031ce01e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java @@ -1,11 +1,55 @@ package com.tencent.supersonic.chat.parser.llm.dsl; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.service.ConfigService; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.common.util.DatePeriodEnum; import com.tencent.supersonic.common.util.DateUtils; +import java.util.List; +import java.util.Objects; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; public class DSLDateHelper { - public static String getCurrentDate(Long modelId) { - return DateUtils.getBeforeDate(4); + public static String getReferenceDate(Long modelId) { + String chatDetailDate = getChatDetailDate(modelId); + if (StringUtils.isNotBlank(chatDetailDate)) { + return chatDetailDate; + } + return DateUtils.getBeforeDate(0); + } + private static String getChatDetailDate(Long modelId) { + if (Objects.isNull(modelId)) { + return null; + } + ChatConfigFilter filter = new ChatConfigFilter(); + filter.setModelId(modelId); + + List configResps = ContextUtils.getBean(ConfigService.class).search(filter, null); + if (CollectionUtils.isEmpty(configResps)) { + return null; + } + ChatConfigResp chatConfigResp = configResps.get(0); + if (Objects.isNull(chatConfigResp.getChatDetailConfig()) || Objects.isNull( + chatConfigResp.getChatDetailConfig().getChatDefaultConfig())) { + return null; + } + + ChatDefaultConfigReq chatDefaultConfig = chatConfigResp.getChatDetailConfig().getChatDefaultConfig(); + Integer unit = chatDefaultConfig.getUnit(); + String period = chatDefaultConfig.getPeriod(); + if (Objects.nonNull(unit)) { + DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(period); + if (Objects.isNull(datePeriodEnum)) { + return DateUtils.getBeforeDate(unit); + } else { + return DateUtils.getBeforeDate(unit, datePeriodEnum); + } + } + return null; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java index c8fa48925..82f3f26b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java @@ -6,11 +6,11 @@ import com.tencent.supersonic.chat.agent.tool.DslTool; import com.tencent.supersonic.chat.api.component.SemanticCorrector; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; @@ -67,7 +67,7 @@ public class LLMDslParser implements SemanticParser { QueryReq request = queryCtx.getRequest(); LLMConfig llmConfig = ContextUtils.getBean(LLMConfig.class); if (StringUtils.isEmpty(llmConfig.getUrl()) || SatisfactionChecker.check(queryCtx)) { - log.info("llmConfig:{}, skip function parser, queryText:{}", llmConfig, request.getQueryText()); + log.info("llmConfig:{}, skip dsl parser, queryText:{}", llmConfig, request.getQueryText()); return; } try { @@ -93,22 +93,56 @@ public class LLMDslParser implements SemanticParser { SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, dslTool, dslParseResult); - CorrectionInfo correctionInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput()); + SemanticCorrectInfo semanticCorrectInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput()); - llmResp.setCorrectorSql(correctionInfo.getSql()); + llmResp.setCorrectorSql(semanticCorrectInfo.getSql()); - setFilter(correctionInfo, modelId, parseInfo); + setFilter(semanticCorrectInfo, modelId, parseInfo); + + setDimensionsAndMetrics(modelId, parseInfo, semanticCorrectInfo.getSql()); } catch (Exception e) { log.error("LLMDSLParser error", e); } } - public void setFilter(CorrectionInfo correctionInfo, Long modelId, SemanticParseInfo parseInfo) { + private void setDimensionsAndMetrics(Long modelId, SemanticParseInfo parseInfo, String sql) { + SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - String correctorSql = correctionInfo.getPreSql(); + if (Objects.isNull(semanticSchema)) { + return; + } + List allFields = getFieldsExceptDate(sql); + + Set metrics = getElements(modelId, allFields, semanticSchema.getMetrics()); + parseInfo.setMetrics(metrics); + + Set dimensions = getElements(modelId, allFields, semanticSchema.getDimensions()); + parseInfo.setDimensions(dimensions); + } + + private Set getElements(Long modelId, List allFields, List elements) { + return elements.stream() + .filter(schemaElement -> modelId.equals(schemaElement.getModel()) + && allFields.contains(schemaElement.getBizName()) + ).collect(Collectors.toSet()); + } + + private List getFieldsExceptDate(String sql) { + List allFields = SqlParserSelectHelper.getAllFields(sql); + if (CollectionUtils.isEmpty(allFields)) { + return new ArrayList<>(); + } + return allFields.stream() + .filter(entry -> !TimeDimensionEnum.getNameList().contains(entry)) + .collect(Collectors.toList()); + } + + public void setFilter(SemanticCorrectInfo semanticCorrectInfo, Long modelId, SemanticParseInfo parseInfo) { + + String correctorSql = semanticCorrectInfo.getPreSql(); if (StringUtils.isEmpty(correctorSql)) { - correctorSql = correctionInfo.getSql(); + correctorSql = semanticCorrectInfo.getSql(); } List expressions = SqlParserSelectHelper.getFilterExpression(correctorSql); if (CollectionUtils.isEmpty(expressions)) { @@ -204,9 +238,9 @@ public class LLMDslParser implements SemanticParser { return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue()); } - private CorrectionInfo getCorrectorSql(QueryContext queryCtx, SemanticParseInfo parseInfo, String sql) { + private SemanticCorrectInfo getCorrectorSql(QueryContext queryCtx, SemanticParseInfo parseInfo, String sql) { - CorrectionInfo correctionInfo = CorrectionInfo.builder() + SemanticCorrectInfo correctInfo = SemanticCorrectInfo.builder() .queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql) .parseInfo(parseInfo).build(); @@ -214,14 +248,13 @@ public class LLMDslParser implements SemanticParser { dslCorrections.forEach(dslCorrection -> { try { - dslCorrection.corrector(correctionInfo); - log.info("sqlCorrection:{} sql:{}", dslCorrection.getClass().getSimpleName(), - correctionInfo.getSql()); + dslCorrection.correct(correctInfo); + log.info("sqlCorrection:{} sql:{}", dslCorrection.getClass().getSimpleName(), correctInfo.getSql()); } catch (Exception e) { - log.error("sqlCorrection:{} execute error,correctionInfo:{}", dslCorrection, correctionInfo, e); + log.error("sqlCorrection:{} correct error,correctInfo:{}", dslCorrection, correctInfo, e); } }); - return correctionInfo; + return correctInfo; } private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, DslTool dslTool, @@ -305,12 +338,12 @@ public class LLMDslParser implements SemanticParser { List linking = new ArrayList<>(); linking.addAll(getValueList(queryCtx, modelId, semanticSchema)); llmReq.setLinking(linking); - String currentDate = DSLDateHelper.getCurrentDate(modelId); + String currentDate = DSLDateHelper.getReferenceDate(modelId); llmReq.setCurrentDate(currentDate); return llmReq; } - private List getValueList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema) { + protected List getValueList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema) { Map itemIdToName = getItemIdToName(modelId, semanticSchema); List matchedElements = queryCtx.getMapInfo().getMatchedElements(modelId); @@ -348,7 +381,7 @@ public class LLMDslParser implements SemanticParser { } - private List getFieldNameList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema) { + protected List getFieldNameList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema) { Map itemIdToName = getItemIdToName(modelId, semanticSchema); List matchedElements = queryCtx.getMapInfo().getMatchedElements(modelId); @@ -375,7 +408,7 @@ public class LLMDslParser implements SemanticParser { return new ArrayList<>(fieldNameList); } - private Map getItemIdToName(Long modelId, SemanticSchema semanticSchema) { + protected Map getItemIdToName(Long modelId, SemanticSchema semanticSchema) { return semanticSchema.getDimensions().stream() .filter(entry -> modelId.equals(entry.getModel())) .collect(Collectors.toMap(SchemaElement::getId, SchemaElement::getName, (value1, value2) -> value2)); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java index 3577a10d6..8965bcda2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.parser.SatisfactionChecker; -import com.tencent.supersonic.chat.query.metricinterpret.MetricInterpretQuery; +import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.service.AgentService; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java index 0d3350cbe..13c038812 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.chat.parser.plugin; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; @@ -8,7 +10,6 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginManager; @@ -16,14 +17,13 @@ import com.tencent.supersonic.chat.plugin.PluginParseResult; import com.tencent.supersonic.chat.plugin.PluginRecallResult; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; -import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import org.springframework.util.CollectionUtils; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public abstract class PluginParser implements SemanticParser { @@ -45,7 +45,11 @@ public abstract class PluginParser implements SemanticParser { public void buildQuery(QueryContext queryContext, PluginRecallResult pluginRecallResult) { Plugin plugin = pluginRecallResult.getPlugin(); - for (Long modelId : pluginRecallResult.getModelIds()) { + Set modelIds = pluginRecallResult.getModelIds(); + if (plugin.isContainsAllModel()) { + modelIds = Sets.newHashSet(-1L); + } + for (Long modelId : modelIds) { PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, plugin, queryContext.getRequest(), queryContext.getMapInfo().getMatchedElements(modelId), pluginRecallResult.getDistance()); @@ -53,9 +57,6 @@ public abstract class PluginParser implements SemanticParser { semanticParseInfo.setScore(pluginRecallResult.getScore()); pluginQuery.setParseInfo(semanticParseInfo); queryContext.getCandidateQueries().add(pluginQuery); - if (plugin.isContainsAllModel()) { - break; - } } } @@ -68,6 +69,9 @@ public abstract class PluginParser implements SemanticParser { if (modelId == null && !CollectionUtils.isEmpty(plugin.getModelList())) { modelId = plugin.getModelList().get(0); } + if (schemaElementMatches == null) { + schemaElementMatches = Lists.newArrayList(); + } SchemaElement model = new SchemaElement(); model.setModel(modelId); model.setId(modelId); @@ -85,17 +89,9 @@ public abstract class PluginParser implements SemanticParser { semanticParseInfo.setProperties(properties); semanticParseInfo.setScore(distance); fillSemanticParseInfo(semanticParseInfo); - setEntity(modelId, semanticParseInfo); return semanticParseInfo; } - private void setEntity(Long modelId, SemanticParseInfo semanticParseInfo) { - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = semanticService.getModelSchema(modelId); - if (modelSchema != null && modelSchema.getEntity() != null) { - semanticParseInfo.setEntity(modelSchema.getEntity()); - } - } private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) { List schemaElementMatches = semanticParseInfo.getElementMatches(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java index 1ca64fe4b..8605560d2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java @@ -28,6 +28,10 @@ public class EmbeddingBasedParser extends PluginParser { if (StringUtils.isBlank(embeddingConfig.getUrl())) { return false; } + List plugins = getPluginList(queryContext); + if (CollectionUtils.isEmpty(plugins)) { + return false; + } List semanticQueries = queryContext.getCandidateQueries(); for (SemanticQuery semanticQuery : semanticQueries) { if (queryContext.getRequest().getQueryText().length() <= semanticQuery.getParseInfo().getScore()) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java index 42c3b3db5..df3bdfc0f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java @@ -11,13 +11,13 @@ public class EmbeddingConfig { @Value("${embedding.url:}") private String url; - @Value("${embedding.recognize.path:preset_query_retrival}") + @Value("${embedding.recognize.path:/preset_query_retrival}") private String recognizePath; - @Value("${embedding.delete.path:preset_delete_by_ids}") + @Value("${embedding.delete.path:/preset_delete_by_ids}") private String deletePath; - @Value("${embedding.add.path:preset_query_add}") + @Value("${embedding.add.path:/preset_query_add}") private String addPath; @Value("${embedding.nResult:1}") diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java index b7c58051e..71f15fd68 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java @@ -44,7 +44,8 @@ public class FunctionBasedParser extends PluginParser { queryContext.getRequest().getQueryText()); return false; } - return true; + List plugins = getPluginList(queryContext); + return !CollectionUtils.isEmpty(plugins); } @Override @@ -82,7 +83,6 @@ public class FunctionBasedParser extends PluginParser { log.info("function call parser, plugin is empty, skip"); return null; } - FunctionCallInfoConfig functionCallConfig = ContextUtils.getBean(FunctionCallInfoConfig.class); FunctionResp functionResp = new FunctionResp(); if (pluginToFunctionCall.size() == 1) { functionResp.setToolSelection(pluginToFunctionCall.iterator().next().getName()); @@ -90,7 +90,7 @@ public class FunctionBasedParser extends PluginParser { FunctionReq functionReq = FunctionReq.builder() .queryText(queryContext.getRequest().getQueryText()) .pluginConfigs(pluginToFunctionCall).build(); - functionResp = requestFunction(functionCallConfig.getUrl(), functionReq); + functionResp = requestFunction(functionReq); } return functionResp; } @@ -133,7 +133,9 @@ public class FunctionBasedParser extends PluginParser { return functionDOList; } - public FunctionResp requestFunction(String url, FunctionReq functionReq) { + public FunctionResp requestFunction(FunctionReq functionReq) { + FunctionCallInfoConfig functionCallInfoConfig = ContextUtils.getBean(FunctionCallInfoConfig.class); + String url = functionCallInfoConfig.getUrl() + functionCallInfoConfig.getPluginSelectPath(); HttpHeaders headers = new HttpHeaders(); long startTime = System.currentTimeMillis(); headers.setContentType(MediaType.APPLICATION_JSON); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java index 7171ad227..317a0fc10 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java @@ -43,6 +43,9 @@ public class AgentCheckParser implements SemanticParser { if (!tool.getQueryModes().contains(query.getQueryMode())) { return true; } + if (CollectionUtils.isEmpty(tool.getModelIds())) { + return true; + } if (tool.isContainsAllModel() || tool.getModelIds().contains(query.getParseInfo().getModelId())) { return false; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java index d2e2d13b1..aa8d13e7d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java @@ -1,5 +1,12 @@ package com.tencent.supersonic.chat.parser.rule; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.MODEL; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; + import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; @@ -8,8 +15,8 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery; import java.util.AbstractMap; import java.util.ArrayList; @@ -21,13 +28,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.MODEL; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; - @Slf4j public class ContextInheritParser implements SemanticParser { @@ -97,10 +97,10 @@ public class ContextInheritParser implements SemanticParser { } protected boolean shouldInherit(QueryContext queryContext, ChatContext chatContext) { - Long contextmodelId = chatContext.getParseInfo().getModelId(); + Long contextModelId = chatContext.getParseInfo().getModelId(); // if map info doesn't contain the same Model of the context, // no inheritance could be done - if (queryContext.getMapInfo().getMatchedElements(contextmodelId) == null) { + if (queryContext.getMapInfo().getMatchedElements(contextModelId) == null) { return false; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java index 536d994a9..09c73ab61 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java @@ -6,8 +6,13 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; + +import java.util.HashMap; import java.util.List; +import java.util.Map; + import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; @Slf4j public class QueryModeParser implements SemanticParser { @@ -24,6 +29,14 @@ public class QueryModeParser implements SemanticParser { queryContext.getCandidateQueries().add(query); } } + // if modelElementMatches id empty,so remove it. + Map> filterModelElementMatches = new HashMap<>(); + for (Long modelId : queryContext.getMapInfo().getModelElementMatches().keySet()) { + if (!CollectionUtils.isEmpty(queryContext.getMapInfo().getModelElementMatches().get(modelId))) { + filterModelElementMatches.put(modelId, queryContext.getMapInfo().getModelElementMatches().get(modelId)); + } + } + queryContext.getMapInfo().setModelElementMatches(filterModelElementMatches); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java index a94621b39..b51c12e3a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java @@ -153,6 +153,7 @@ public class TimeRangeParser implements SemanticParser { } info.setDetectWord(detectWord); info.setStartDate(LocalDate.now().minusDays(days).toString()); + info.setEndDate(LocalDate.now().minusDays(1).toString()); info.setUnit(num); return info; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java index c330e53ff..c82980567 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java @@ -55,4 +55,8 @@ public class Plugin extends RecordInfo { return CollectionUtils.isNotEmpty(modelList) && modelList.contains(-1L); } + public Long getDefaultMode() { + return -1L; + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java index 156f3edab..0c8429bfd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java @@ -15,6 +15,7 @@ import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp; import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval; import com.tencent.supersonic.chat.plugin.event.PluginAddEvent; +import com.tencent.supersonic.chat.plugin.event.PluginDelEvent; import com.tencent.supersonic.chat.plugin.event.PluginUpdateEvent; import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.WebBase; @@ -116,8 +117,8 @@ public class PluginManager { } @EventListener - public void delPlugin(PluginAddEvent pluginAddEvent) { - Plugin plugin = pluginAddEvent.getPlugin(); + public void delPlugin(PluginDelEvent pluginDelEvent) { + Plugin plugin = pluginDelEvent.getPlugin(); if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) { requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(plugin))); } @@ -142,18 +143,22 @@ public class PluginManager { return ResponseEntity.of(Optional.empty()); } String url = embeddingConfig.getUrl() + path; - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - headers.setLocation(URI.create(url)); - URI requestUrl = UriComponentsBuilder - .fromHttpUrl(url).build().encode().toUri(); - HttpEntity entity = new HttpEntity<>(jsonBody, headers); - log.info("[embedding] equest body :{}, url:{}", jsonBody, url); - ResponseEntity responseEntity = - restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference() { - }); - log.info("[embedding] result body:{}", responseEntity); - return responseEntity; + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setLocation(URI.create(url)); + URI requestUrl = UriComponentsBuilder + .fromHttpUrl(url).build().encode().toUri(); + HttpEntity entity = new HttpEntity<>(jsonBody, headers); + log.info("[embedding] equest body :{}, url:{}", jsonBody, url); + ResponseEntity responseEntity = restTemplate.exchange(requestUrl, + HttpMethod.POST, entity, new ParameterizedTypeReference() {}); + log.info("[embedding] result body:{}", responseEntity); + return responseEntity; + } catch (Throwable e) { + log.warn("connect to embedding service failed, url:{}", url); + } + return ResponseEntity.of(Optional.empty()); } public void requestEmbeddingPluginAddALL(List plugins) { @@ -298,7 +303,7 @@ public class PluginManager { private static Set getPluginMatchedModel(Plugin plugin, QueryContext queryContext) { Set matchedModel = queryContext.getMapInfo().getMatchedModels(); if (plugin.isContainsAllModel()) { - return matchedModel; + return Sets.newHashSet(plugin.getDefaultMode()); } List modelIds = plugin.getModelList(); Set pluginMatchedModel = Sets.newHashSet(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerReq.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerReq.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerReq.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerReq.java index 6f996b587..4a0ba3c2c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerReq.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.query.metricinterpret; +package com.tencent.supersonic.chat.query.llm.interpret; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerResp.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerResp.java similarity index 62% rename from chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerResp.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerResp.java index 8a10b25c7..44c72b82a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerResp.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerResp.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.query.metricinterpret; +package com.tencent.supersonic.chat.query.llm.interpret; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/MetricInterpretQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java similarity index 99% rename from chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/MetricInterpretQuery.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java index 49f8b444e..1fa090eb8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/MetricInterpretQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.query.metricinterpret; +package com.tencent.supersonic.chat.query.llm.interpret; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java index d03ae402b..c8d75016e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.ContextUtils; @@ -56,9 +57,18 @@ public class WebServiceQuery extends PluginSemanticQuery { PluginParseResult pluginParseResult = JsonUtil.toObject( JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class); WebServiceResponse webServiceResponse = buildResponse(pluginParseResult); - queryResult.setResponse(webServiceResponse); - queryResult.setQueryState(QueryState.SUCCESS); - //parseInfo.setProperties(null); + Object object = webServiceResponse.getResult(); + // in order to show webServiceQuery result int frontend conveniently, + // webServiceResponse result format is consistent with queryByStruct result. + log.info("webServiceResponse result:{}", JsonUtil.toString(object)); + try { + Map data = JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class); + queryResult.setQueryResults((List>) data.get("resultList")); + queryResult.setQueryColumns((List) data.get("columns")); + queryResult.setQueryState(QueryState.SUCCESS); + } catch (Exception e) { + log.info("webServiceResponse result has an exception:{}", e.getMessage()); + } return queryResult; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java index fa92f91ea..d8348c077 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -141,61 +142,44 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { if (!id2Values.isEmpty()) { for (Map.Entry> entry : id2Values.entrySet()) { - SchemaElement entity = modelSchema.getElement(SchemaElementType.ENTITY, entry.getKey()); - - if (entry.getValue().size() == 1) { - SchemaElementMatch schemaMatch = entry.getValue().get(0); - QueryFilter dimensionFilter = new QueryFilter(); - dimensionFilter.setValue(schemaMatch.getWord()); - dimensionFilter.setBizName(entity.getBizName()); - dimensionFilter.setName(entity.getName()); - dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); - dimensionFilter.setElementID(schemaMatch.getElement().getId()); - parseInfo.getDimensionFilters().add(dimensionFilter); - parseInfo.setEntity(modelSchema.getEntity()); - } else { - QueryFilter dimensionFilter = new QueryFilter(); - List vals = new ArrayList<>(); - entry.getValue().stream().forEach(i -> vals.add(i.getWord())); - dimensionFilter.setValue(vals); - dimensionFilter.setBizName(entity.getBizName()); - dimensionFilter.setName(entity.getName()); - dimensionFilter.setOperator(FilterOperatorEnum.IN); - dimensionFilter.setElementID(entry.getKey()); - parseInfo.getDimensionFilters().add(dimensionFilter); - } + addFilters(parseInfo, modelSchema, entry, SchemaElementType.ENTITY); } } if (!dim2Values.isEmpty()) { for (Map.Entry> entry : dim2Values.entrySet()) { - SchemaElement dimension = modelSchema.getElement(SchemaElementType.DIMENSION, entry.getKey()); - - if (entry.getValue().size() == 1) { - SchemaElementMatch schemaMatch = entry.getValue().get(0); - QueryFilter dimensionFilter = new QueryFilter(); - dimensionFilter.setValue(schemaMatch.getWord()); - dimensionFilter.setBizName(dimension.getBizName()); - dimensionFilter.setName(dimension.getName()); - dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); - dimensionFilter.setElementID(schemaMatch.getElement().getId()); - parseInfo.getDimensionFilters().add(dimensionFilter); - parseInfo.setEntity(modelSchema.getEntity()); - } else { - QueryFilter dimensionFilter = new QueryFilter(); - List vals = new ArrayList<>(); - entry.getValue().stream().forEach(i -> vals.add(i.getWord())); - dimensionFilter.setValue(vals); - dimensionFilter.setBizName(dimension.getBizName()); - dimensionFilter.setName(dimension.getName()); - dimensionFilter.setOperator(FilterOperatorEnum.IN); - dimensionFilter.setElementID(entry.getKey()); - parseInfo.getDimensionFilters().add(dimensionFilter); - } + addFilters(parseInfo, modelSchema, entry, SchemaElementType.DIMENSION); } } } + private void addFilters(SemanticParseInfo parseInfo, ModelSchema modelSchema, + Entry> entry, SchemaElementType dimension1) { + SchemaElement dimension = modelSchema.getElement(dimension1, entry.getKey()); + + if (entry.getValue().size() == 1) { + SchemaElementMatch schemaMatch = entry.getValue().get(0); + QueryFilter dimensionFilter = new QueryFilter(); + dimensionFilter.setValue(schemaMatch.getWord()); + dimensionFilter.setBizName(dimension.getBizName()); + dimensionFilter.setName(dimension.getName()); + dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); + dimensionFilter.setElementID(schemaMatch.getElement().getId()); + parseInfo.getDimensionFilters().add(dimensionFilter); + parseInfo.setEntity(modelSchema.getEntity()); + } else { + QueryFilter dimensionFilter = new QueryFilter(); + List vals = new ArrayList<>(); + entry.getValue().stream().forEach(i -> vals.add(i.getWord())); + dimensionFilter.setValue(vals); + dimensionFilter.setBizName(dimension.getBizName()); + dimensionFilter.setName(dimension.getName()); + dimensionFilter.setOperator(FilterOperatorEnum.IN); + dimensionFilter.setElementID(entry.getKey()); + parseInfo.getDimensionFilters().add(dimensionFilter); + } + } + @Override public QueryResult execute(User user) { @@ -292,7 +276,6 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { } - protected QueryStructReq convertQueryStruct() { return QueryReqBuilder.buildStructReq(parseInfo); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/DictionaryController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java similarity index 70% rename from chat/core/src/main/java/com/tencent/supersonic/chat/rest/DictionaryController.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java index 09fb97995..1cbf74fff 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/DictionaryController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java @@ -4,29 +4,35 @@ package com.tencent.supersonic.chat.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.service.DictionaryService; +import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener; import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand; import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; + import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PutMapping; @RestController @RequestMapping("/api/chat/dict") -public class DictionaryController { +public class KnowledgeController { @Autowired private DictionaryService dictApplicationService; + @Autowired + private ApplicationStartedListener applicationStartedListener; + /** * addDictInfo * @@ -34,8 +40,8 @@ public class DictionaryController { */ @PostMapping("/task") public Long addDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dictApplicationService.addDictTask(dimValue2DictCommend, user); } @@ -47,8 +53,8 @@ public class DictionaryController { */ @DeleteMapping("/task") public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dictApplicationService.deleteDictTask(dimValue2DictCommend, user); } @@ -60,16 +66,22 @@ public class DictionaryController { */ @PostMapping("/task/search") public List searchDictTaskList(@RequestBody DictTaskFilter filter, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return dictApplicationService.searchDictTaskList(filter, user); } @GetMapping("/rootPath") public String getDictRootPath(HttpServletRequest request, - HttpServletResponse response) { + HttpServletResponse response) { return dictApplicationService.getDictRootPath(); } -} \ No newline at end of file + @PutMapping("/knowledge/dimValue") + public Boolean updateDimValue(HttpServletRequest request, + HttpServletResponse response) { + return applicationStartedListener.updateKnowledgeDimValue(); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java index 685b5cbfd..0945633b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java @@ -5,8 +5,10 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.config.ChatConfig; import java.util.List; @@ -16,6 +18,10 @@ public interface ConfigService { Long editConfig(ChatConfigEditReqReq extendEditCmd, User user); + ItemNameVisibilityInfo getItemNameVisibility(ChatConfig chatConfig); + + ItemNameVisibilityInfo getVisibilityByModelId(Long modelId); + List search(ChatConfigFilter filter, User user); ChatConfigRichResp getConfigRichInfo(Long modelId); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java index 49a40f615..a97d3a404 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java @@ -5,15 +5,16 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; -import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; -import com.tencent.supersonic.chat.api.pojo.request.Entity; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; +import com.tencent.supersonic.chat.api.pojo.request.Entity; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; @@ -27,6 +28,7 @@ import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository; import com.tencent.supersonic.chat.utils.ChatConfigHelper; +import com.tencent.supersonic.chat.utils.VisibilityEvent; import com.tencent.supersonic.common.util.JsonUtil; import java.util.ArrayList; @@ -36,9 +38,14 @@ import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.model.domain.DimensionService; +import com.tencent.supersonic.semantic.model.domain.MetricService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -49,16 +56,24 @@ public class ConfigServiceImpl implements ConfigService { private final ChatConfigRepository chatConfigRepository; private final ChatConfigHelper chatConfigHelper; + private final DimensionService dimensionService; + private final MetricService metricService; @Autowired private SemanticService semanticService; + @Autowired + private ApplicationEventPublisher applicationEventPublisher; private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer(); public ConfigServiceImpl(ChatConfigRepository chatConfigRepository, - ChatConfigHelper chatConfigHelper) { + ChatConfigHelper chatConfigHelper, + DimensionService dimensionService, + MetricService metricService) { this.chatConfigRepository = chatConfigRepository; this.chatConfigHelper = chatConfigHelper; + this.dimensionService = dimensionService; + this.metricService = metricService; } @Override @@ -68,6 +83,7 @@ public class ConfigServiceImpl implements ConfigService { permissionCheckLogic(configBaseCmd.getModelId(), user.getName()); ChatConfig chaConfig = chatConfigHelper.newChatConfig(configBaseCmd, user); Long id = chatConfigRepository.createConfig(chaConfig); + applicationEventPublisher.publishEvent(new VisibilityEvent(this, chaConfig)); return id; } @@ -91,9 +107,56 @@ public class ConfigServiceImpl implements ConfigService { permissionCheckLogic(configEditCmd.getModelId(), user.getName()); ChatConfig chaConfig = chatConfigHelper.editChatConfig(configEditCmd, user); chatConfigRepository.updateConfig(chaConfig); + applicationEventPublisher.publishEvent(new VisibilityEvent(this, chaConfig)); return configEditCmd.getId(); } + public ItemNameVisibilityInfo getVisibilityByModelId(Long modelId) { + ChatConfigResp chatConfigResp = fetchConfigByModelId(modelId); + ChatConfig chatConfig = new ChatConfig(); + chatConfig.setModelId(modelId); + chatConfig.setChatAggConfig(chatConfigResp.getChatAggConfig()); + chatConfig.setChatDetailConfig(chatConfigResp.getChatDetailConfig()); + ItemNameVisibilityInfo itemNameVisibility = getItemNameVisibility(chatConfig); + return itemNameVisibility; + } + + public ItemNameVisibilityInfo getItemNameVisibility(ChatConfig chatConfig) { + Long modelId = chatConfig.getModelId(); + + List blackDimIdList = new ArrayList<>(); + if (Objects.nonNull(chatConfig.getChatAggConfig()) && Objects.nonNull(chatConfig.getChatAggConfig())) { + blackDimIdList.addAll(chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList()); + } + if (Objects.nonNull(chatConfig.getChatDetailConfig()) && Objects.nonNull(chatConfig.getChatDetailConfig())) { + blackDimIdList.addAll(chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList()); + } + List filterDimIdList = blackDimIdList.stream().distinct().collect(Collectors.toList()); + + List blackMetricIdList = new ArrayList<>(); + if (Objects.nonNull(chatConfig.getChatAggConfig()) && Objects.nonNull(chatConfig.getChatAggConfig())) { + blackMetricIdList.addAll(chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList()); + } + if (Objects.nonNull(chatConfig.getChatDetailConfig()) && Objects.nonNull(chatConfig.getChatDetailConfig())) { + blackMetricIdList.addAll(chatConfig.getChatDetailConfig().getVisibility().getBlackMetricIdList()); + } + List filterMetricIdList = blackMetricIdList.stream().distinct().collect(Collectors.toList()); + + ItemNameVisibilityInfo itemNameVisibility = new ItemNameVisibilityInfo(); + if (!CollectionUtils.isEmpty(blackDimIdList)) { + List dimensionRespList = dimensionService.getDimensions(modelId); + List blackDimNameList = dimensionRespList.stream().filter(o -> filterDimIdList.contains(o.getId())) + .map(o -> o.getName()).collect(Collectors.toList()); + itemNameVisibility.setBlackDimNameList(blackDimNameList); + } + if (!CollectionUtils.isEmpty(blackMetricIdList)) { + List metricRespList = metricService.getMetrics(modelId); + List blackMetricList = metricRespList.stream().filter(o -> filterMetricIdList.contains(o.getId())) + .map(o -> o.getName()).collect(Collectors.toList()); + itemNameVisibility.setBlackMetricNameList(blackMetricList); + } + return itemNameVisibility; + } /** * model administrators have the right to modify related configuration information. diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index de92800e5..0e1a1e59f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -247,7 +247,9 @@ public class QueryServiceImpl implements QueryService { public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException { SemanticQuery semanticQuery = QueryManager.createRuleQuery(queryData.getQueryMode()); BeanUtils.copyProperties(queryData, semanticQuery.getParseInfo()); - return semanticQuery.execute(user); + QueryResult queryResult = semanticQuery.execute(user); + queryResult.setChatContext(semanticQuery.getParseInfo()); + return queryResult; } @Override @@ -274,7 +276,7 @@ public class QueryServiceImpl implements QueryService { List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); queryStructReq.setGroups(groups); - if (Objects.isNull(dimensionValueReq.getValue())) { + if (!Objects.isNull(dimensionValueReq.getValue())) { List dimensionFilters = new ArrayList<>(); Filter dimensionFilter = new Filter(); dimensionFilter.setOperator(FilterOperatorEnum.LIKE); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java index 9828d2581..4a6827b82 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java @@ -1,16 +1,19 @@ package com.tencent.supersonic.chat.service.impl; +import com.github.benmanes.caffeine.cache.Cache; import com.google.common.collect.Lists; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.SearchResult; import com.tencent.supersonic.chat.mapper.MapperHelper; +import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.knowledge.dictionary.ModelInfoStat; import com.tencent.supersonic.chat.mapper.ModelWithSemanticType; @@ -22,7 +25,7 @@ import com.tencent.supersonic.chat.service.SearchService; import com.tencent.supersonic.knowledge.utils.NatureHelper; import com.tencent.supersonic.knowledge.dictionary.DictWord; import com.tencent.supersonic.knowledge.dictionary.MapResult; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.knowledge.utils.HanlpHelper; import java.util.ArrayList; @@ -40,6 +43,7 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -59,6 +63,12 @@ public class SearchServiceImpl implements SearchService { private SearchMatchStrategy searchMatchStrategy; @Autowired private AgentService agentService; + @Autowired + @Qualifier("searchCaffeineCache") + private Cache caffeineCache; + + @Autowired + private ConfigService configService; @Override public List search(QueryReq queryReq) { @@ -80,7 +90,7 @@ public class SearchServiceImpl implements SearchService { // 3.detect by segment List originals = HanlpHelper.getTerms(queryText); - + log.info("hanlp parse result: {}", originals); MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); Set detectModelIds = mapperHelper.getModelIds(queryReq); @@ -94,7 +104,6 @@ public class SearchServiceImpl implements SearchService { .reduce((entry1, entry2) -> entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length() ? entry1 : entry2); - log.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult); // 5.optimize the results after the query if (!mostSimilarSearchResult.isPresent()) { @@ -194,7 +203,16 @@ public class SearchServiceImpl implements SearchService { } searchResults.add(searchResult); int metricSize = getMetricSize(natureToNameMap); - List metrics = filerMetricsByModel(metricsDb, modelId, metricSize); + //invisibility to filter metrics + ItemNameVisibilityInfo itemNameVisibility = (ItemNameVisibilityInfo) caffeineCache.getIfPresent(modelId); + if (itemNameVisibility == null) { + itemNameVisibility = configService.getVisibilityByModelId(modelId); + caffeineCache.put(modelId, itemNameVisibility); + } + List blackMetricNameList = itemNameVisibility.getBlackMetricNameList(); + List metrics = filerMetricsByModel(metricsDb, modelId, metricSize * 3) + .stream().filter(o -> !blackMetricNameList.contains(o)) + .limit(metricSize).collect(Collectors.toList()); for (String metric : metrics) { SearchResult result = SearchResult.builder() @@ -279,7 +297,7 @@ public class SearchServiceImpl implements SearchService { private boolean searchMetricAndDimension(Set possibleModels, Map modelToName, Map.Entry> searchTextEntry, Set searchResults) { boolean existMetric = false; - + log.info("searchMetricAndDimension searchTextEntry:{}", searchTextEntry); MatchText matchText = searchTextEntry.getKey(); List mapResults = searchTextEntry.getValue(); @@ -297,7 +315,6 @@ public class SearchServiceImpl implements SearchService { existMetric = true; Long modelId = modelWithSemanticType.getModel(); SchemaElementType semanticType = modelWithSemanticType.getSemanticType(); - SearchResult searchResult = SearchResult.builder() .modelId(modelId) .modelName(modelToName.get(modelId)) @@ -305,12 +322,25 @@ public class SearchServiceImpl implements SearchService { .subRecommend(mapResult.getName()) .schemaElementType(semanticType) .build(); - - searchResults.add(searchResult); + //visibility to filter metrics + ItemNameVisibilityInfo visibility = (ItemNameVisibilityInfo) caffeineCache.getIfPresent(modelId); + if (visibility == null) { + visibility = configService.getVisibilityByModelId(modelId); + caffeineCache.put(modelId, visibility); + } + if (semanticType.equals(SchemaElementType.DIMENSION) + && !visibility.getBlackDimNameList().contains(mapResult.getName())) { + searchResults.add(searchResult); + } + if (semanticType.equals(SchemaElementType.METRIC) + && !visibility.getBlackMetricNameList().contains(mapResult.getName())) { + searchResults.add(searchResult); + } } log.info("parseResult:{},dimensionMetricClassIds:{},possibleModels:{}", mapResult, dimensionMetricClassIds, possibleModels); } + log.info("searchMetricAndDimension searchResults:{}", searchResults); return existMetric; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/VisibilityEvent.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/VisibilityEvent.java new file mode 100644 index 000000000..ca973b6ee --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/VisibilityEvent.java @@ -0,0 +1,22 @@ +package com.tencent.supersonic.chat.utils; + +import com.tencent.supersonic.chat.config.ChatConfig; +import org.springframework.context.ApplicationEvent; + +public class VisibilityEvent extends ApplicationEvent { + private static final long serialVersionUID = 1L; + private ChatConfig chatConfig; + + public VisibilityEvent(Object source, ChatConfig chatConfig) { + super(source); + this.chatConfig = chatConfig; + } + + public void setChatConfig(ChatConfig chatConfig) { + this.chatConfig = chatConfig; + } + + public ChatConfig getChatConfig() { + return chatConfig; + } +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/VisibilityListener.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/VisibilityListener.java new file mode 100644 index 000000000..7e227e4b5 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/VisibilityListener.java @@ -0,0 +1,30 @@ +package com.tencent.supersonic.chat.utils; + +import com.github.benmanes.caffeine.cache.Cache; +import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; +import com.tencent.supersonic.chat.service.ConfigService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class VisibilityListener implements ApplicationListener { + + @Autowired + @Qualifier("searchCaffeineCache") + private Cache caffeineCache; + + @Autowired + private ConfigService configService; + + @Override + public void onApplicationEvent(VisibilityEvent event) { + log.info("visibility has changed,so update cache!"); + ItemNameVisibilityInfo itemNameVisibility = configService.getItemNameVisibility(event.getChatConfig()); + log.info("itemNameVisibility :{}", itemNameVisibility); + caffeineCache.put(event.getChatConfig().getModelId(), itemNameVisibility); + } +} diff --git a/chat/core/src/main/python/llm/api_service.py b/chat/core/src/main/python/llm/api_service.py index c06748b74..a5bd975f2 100644 --- a/chat/core/src/main/python/llm/api_service.py +++ b/chat/core/src/main/python/llm/api_service.py @@ -25,31 +25,20 @@ app = FastAPI() @app.post("/query2sql/") async def din_query2sql(query_body: Mapping[str, Any]): - if 'queryText' not in query_body: - raise HTTPException(status_code=400, + if 'queryText' not in query_body: + raise HTTPException(status_code=400, detail="query_text is not in query_body") - else: - query_text = query_body['queryText'] + else: + query_text = query_body['queryText'] - if 'schema' not in query_body: - raise HTTPException(status_code=400, detail="schema is not in query_body") - else: - schema = query_body['schema'] + if 'schema' not in query_body: + raise HTTPException(status_code=400, detail="schema is not in query_body") + else: + schema = query_body['schema'] - if 'currentDate' not in query_body: - raise HTTPException(status_code=400, detail="currentDate is not in query_body") - else: - current_date = query_body['currentDate'] + resp = query2sql(query_text=query_text, schema=schema) - if 'linking' not in query_body: - linking = None - else: - linking = query_body['linking'] - - resp = query2sql(query_text=query_text, - schema=schema, current_date=current_date, linking=linking) - - return resp + return resp @app.post("/preset_query_retrival/") diff --git a/chat/core/src/main/python/llm/few_shot_example/sql_exampler.py b/chat/core/src/main/python/llm/few_shot_example/sql_exampler.py index 87a32dc3d..990dc8ec8 100644 --- a/chat/core/src/main/python/llm/few_shot_example/sql_exampler.py +++ b/chat/core/src/main/python/llm/few_shot_example/sql_exampler.py @@ -1,296 +1,147 @@ -examplars= [ - { "current_date":"2020-12-01", +examplars= [ + { "table_name":"内容库产品", - "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""", - "question":"比较jackjchen和robinlee在内容库的访问次数", - "prior_schema_links":"""['jackjchen'->用户名, 'robinlee'->用户名]""", - "analysis": """让我们一步一步地思考。在问题“比较jackjchen和robinlee在内容库的访问次数“中，我们被问： -“比较jackjchen和robinlee”，所以我们需要column=[用户名] -”内容库的访问次数“，所以我们需要column=[访问次数] -基于table和columns，可能的cell values 是 = ['jackjchen', 'robinlee']。""", - "schema_links":"""["用户名", "访问次数", "'jackjchen'", "'robinlee'"]""", - "sql":"""select 用户名, 访问次数 from 内容库产品 where 用户名 in ('jackjchen', 'robinlee') and 数据日期 = '2020-12-01' """ - }, - { "current_date":"2022-11-06", + "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长"]""", + "question":"比较jerry和tom在内容库的访问次数", + "analysis": """让我们一步一步地思考。在问题“比较jerry和tom在内容库的访问次数“中，我们被问： +“内容库的访问次数”，所以我们需要column=[访问次数] +”比较jerry和tom“，所以我们需要column=[用户名] +基于table和columns，可能的cell values 是 = ['jerry', 'tom']。""", + "schema_links":"""["访问次数", "用户名", "'jerry'", "'tom'"]""", + "sql":"""select 用户名, 访问次数 from 内容库产品 where 用户名 in ('jerry', 'tom')""" + }, + { "table_name":"内容库产品", - "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""", + "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长"]""", "question":"内容库近12个月访问人数 按部门", - "prior_schema_links":"""[]""", "analysis": """让我们一步一步地思考。在问题“内容库近12个月访问人数 按部门“中，我们被问： -”内容库近12个月“，所以我们需要column=[数据日期] -“访问人数”，所以我们需要column=[访问人数] +“内容库近12个月访问人数”，所以我们需要column=[访问人数] ”按部门“，所以我们需要column=[部门] -基于table和columns，可能的cell values 是 = [12]。""", - "schema_links":"""["访问人数", "部门", "数据日期", 12]""", - "sql":"""select 部门, 数据日期, 访问人数 from 内容库产品 where datediff('month', 数据日期, '2022-11-06') <= 12 """ - }, - { "current_date":"2023-04-21", +基于table和columns，可能的cell values 是 = []。""", + "schema_links":"""["访问人数", "部门"]""", + "sql":"""select 部门, sum(访问人数) from 内容库产品 where 部门 group by 部门""" + }, + { "table_name":"内容库产品", - "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""", - "question":"内容库美术部、技术研发部的访问时长", - "prior_schema_links":"""['美术部'->部门, '技术研发部'->部门]""", - "analysis": """让我们一步一步地思考。在问题“内容库美术部、技术研发部的访问时长“中，我们被问： + "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长"]""", + "question":"内容库编辑部、美术部的访问时长", + "analysis": """让我们一步一步地思考。在问题“内容库编辑部、美术部的访问时长“中，我们被问： “访问时长”，所以我们需要column=[访问时长] -”内容库美术部、技术研发部“，所以我们需要column=[部门] -基于table和columns，可能的cell values 是 = ['美术部', '技术研发部']。""", - "schema_links":"""["访问时长", "部门", "'美术部'", "'技术研发部'"]""", - "sql":"""select 部门, 访问时长 from 内容库产品 where 部门 in ('美术部', '技术研发部') and 数据日期 = '2023-04-21' """ - }, - { "current_date":"2023-08-21", - "table_name":"严选", - "fields_list":"""["严选版权归属系", "付费模式", "结算播放份额", "付费用户结算播放份额", "数据日期"]""", - "question":"近3天海田飞系MPPM结算播放份额", - "prior_schema_links":"""['海田飞系'->严选版权归属系]""", - "analysis": """让我们一步一步地思考。在问题“近3天海田飞系MPPM结算播放份额“中，我们被问： -“MPPM结算播放份额”，所以我们需要column=[结算播放份额] -”海田飞系“，所以我们需要column=[严选版权归属系] -”近3天“，所以我们需要column=[数据日期] -基于table和columns，可能的cell values 是 = ['海田飞系', 3]。""", - "schema_links":"""["结算播放份额", "严选版权归属系", "数据日期", "'海田飞系'", 3]""", - "sql":"""select 严选版权归属系, 结算播放份额 from 严选 where 严选版权归属系 = '海田飞系' and datediff('day', 数据日期, '2023-08-21') <= 3 """ - }, - { "current_date":"2023-05-22", +”内容库编辑部、美术部“，所以我们需要column=[部门] +基于table和columns，可能的cell values 是 = ['编辑部', '美术部']。""", + "schema_links":"""["访问时长", "部门", "'编辑部'", "'美术部'"]""", + "sql":"""select 部门, 访问时长 from 内容库产品 where 部门 in ('编辑部', '美术部')""" + }, + { + "table_name":"精选", + "fields_list":"""['归属系', '付费模式', '结算播放份额', '付费用户结算播放份额']""", + "question":"近3天飞天系结算播放份额", + "analysis": """让我们一步一步地思考。在问题“近3天飞天系结算播放份额“中，我们被问： +“结算播放份额”，所以我们需要column=[结算播放份额] +”飞天系“，所以我们需要column=[归属系] +基于table和columns，可能的cell values 是 = ['飞天系']。""", + "schema_links":"""["结算播放份额", "归属系", "'飞天系'"]""", + "sql":"""select 归属系, 结算播放份额 from 精选 where 归属系 in ('飞天系')""" + }, + { "table_name":"歌曲库", - "fields_list":"""["是否潮流人歌曲", "C音歌曲ID", "C音歌曲MID", "歌曲名", "歌曲版本", "语种", "歌曲类型", "翻唱类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "结算播放量", "运营播放量", "付费用户结算播放量", "历史累计结算播放量", "运营搜播量", "结算搜播量", "运营完播量", "运营推播量", "近7日复播率", "日均搜播量", "数据日期"]""", - "question":"对比近7天翻唱版和纯音乐的歌曲播放量", - "prior_schema_links":"""['纯音乐'->语种, '翻唱版'->歌曲版本]""", + "fields_list":"""['歌曲ID', '歌曲MID', '歌曲名', '歌曲版本', '歌曲类型', '翻唱类型', '结算播放量', '运营播放量', '付费用户结算播放量', '历史累计结算播放量', '运营搜播量', '结算搜播量', '运营完播量', '运营推播量', '近7日复播率', '日均搜播量']""", + "question":"对比近3天翻唱版和纯音乐的歌曲播放量", "analysis": """让我们一步一步地思考。在问题“对比近3天翻唱版和纯音乐的歌曲播放量“中，我们被问： “歌曲播放量”，所以我们需要column=[结算播放量] -”翻唱版“，所以我们需要column=[歌曲版本] -”和纯音乐的歌曲“，所以我们需要column=[语种] -”近7天“，所以我们需要column=[数据日期] -基于table和columns，可能的cell values 是 = ['翻唱版', '纯音乐', 7]。""", - "schema_links":"""["结算播放量", "歌曲版本", "语种", "数据日期", "'翻唱版'", "'纯音乐'", 7]""", - "sql":"""select 歌曲版本, 语种, 结算播放量 from 歌曲库 where 歌曲版本 = '翻唱版' and 语种 = '纯音乐' and datediff('day', 数据日期, '2023-05-22') <= 7 """ - }, - { "current_date":"2023-05-31", +”翻唱版和纯音乐“，所以我们需要column=[歌曲类型] +基于table和columns，可能的cell values 是 = ['翻唱版', '纯音乐']。""", + "schema_links":"""["结算播放量", "歌曲类型", "'翻唱版'", "'纯音乐'"]""", + "sql":"""select 歌曲类型, 结算播放量 from 歌曲库 where 歌曲类型 in ('翻唱版', '纯音乐')""" + }, + { "table_name":"艺人库", - "fields_list":"""["上下架状态", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "活跃区域", "年龄", "歌手才能", "歌手风格", "粉丝数", "潮音粉丝数", "超声波粉丝数", "推博粉丝数", "超声波歌曲数", "在架歌曲数", "超声波分享数", "独占歌曲数", "超声波在架歌曲评论数", "有播放量歌曲数", "数据日期"]""", - "question":"对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数", - "prior_schema_links":"""['1527896'->MPPM歌手ID, '1565463'->MPPM歌手ID, '2141459'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“对比一下陈拙悬、孟梅琦、赖媚韵的粉丝数“中，我们被问： + "fields_list":"""['上下架状态', '歌手名', '歌手等级', '歌手类型', '歌手来源', '活跃区域', '年龄', '歌手才能', '歌手风格', '粉丝数', '在架歌曲数', '有播放量歌曲数']""", + "question":"对比一下流得滑、锅富程、章雪友的粉丝数", + "analysis": """让我们一步一步地思考。在问题“对比一下流得滑、锅富程、章雪友的粉丝数“中，我们被问： “粉丝数”，所以我们需要column=[粉丝数] -”陈拙悬、孟梅琦、赖媚韵“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = ['陈拙悬', '孟梅琦', '赖媚韵']。""", - "schema_links":"""["粉丝数", "歌手名", "'陈拙悬'", "'孟梅琦'", "'赖媚韵'"]""", - "sql":"""select 歌手名, 粉丝数 from 艺人库 where 歌手名 in ('陈拙悬', '孟梅琦', '赖媚韵') and 数据日期 = '2023-05-31' """ - }, - { "current_date":"2023-07-31", +”流得滑、锅富程、章雪友“，所以我们需要column=[歌手名] +基于table和columns，可能的cell values 是 = ['流得滑', '锅富程', '章雪友']。""", + "schema_links":"""["粉丝数", "歌手名", "'流得滑'", "'锅富程'", "'章雪友'"]""", + "sql":"""select 歌手名, 粉丝数 from 艺人库 where 歌手名 in ('流得滑', '锅富程', '章雪友')""" + }, + { "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", + "fields_list":"""['歌曲ID', '歌曲MID', '歌曲名', '歌曲版本', '歌曲类型', '翻唱类型', '结算播放量', '运营播放量', '付费用户结算播放量', '历史累计结算播放量', '运营搜播量', '结算搜播量', '运营完播量', '运营推播量', '近7日复播率', '日均搜播量']""", "question":"播放量大于1万的歌曲有多少", - "prior_schema_links":"""[]""", "analysis": """让我们一步一步地思考。在问题“播放量大于1万的歌曲有多少“中，我们被问： “歌曲有多少”，所以我们需要column=[歌曲名] -”播放量大于1万的“，所以我们需要column=[结算播放量] +”播放量大于1万“，所以我们需要column=[结算播放量] 基于table和columns，可能的cell values 是 = [10000]。""", "schema_links":"""["歌曲名", "结算播放量", 10000]""", - "sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 10000 and 数据日期 = '2023-07-31' """ - }, - { "current_date":"2023-07-31", + "sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 10000""" + }, + { "table_name":"内容库产品", - "fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""", + "fields_list":"""['用户名', '部门', '模块', '访问时长', '访问次数', '访问人数']""", "question":"内容库访问时长小于1小时，且来自美术部的用户是哪些", - "prior_schema_links":"""['美术部'->部门]""", "analysis": """让我们一步一步地思考。在问题“内容库访问时长小于1小时，且来自美术部的用户是哪些“中，我们被问： “用户是哪些”，所以我们需要column=[用户名] ”美术部的“，所以我们需要column=[部门] ”访问时长小于1小时“，所以我们需要column=[访问时长] 基于table和columns，可能的cell values 是 = ['美术部', 1]。""", "schema_links":"""["用户名", "部门", "访问时长", "'美术部'", 1]""", - "sql":"""select 用户名 from 内容库产品 where 部门 = '美术部' and 访问时长 < 1 and 数据日期 = '2023-07-31' """ - }, - { "current_date":"2023-08-31", + "sql":"""select 用户名 from 内容库产品 where 部门 = '美术部' and 访问时长 < 1""" + }, + { "table_name":"内容库产品", - "fields_list":"""["用户名", "部门", "模块", "访问时长", "访问次数", "访问人数", "数据日期"]""", + "fields_list":"""['用户名', '部门', '模块', '访问次数', '访问人数', '访问时长']""", "question":"内容库pv最高的用户有哪些", - "prior_schema_links":"""[]""", "analysis": """让我们一步一步地思考。在问题“内容库pv最高的用户有哪些“中，我们被问： “用户有哪些”，所以我们需要column=[用户名] ”pv最高的“，所以我们需要column=[访问次数] 基于table和columns，可能的cell values 是 = []。""", "schema_links":"""["用户名", "访问次数"]""", - "sql":"""select 用户名 from 内容库产品 where 数据日期 = '2023-08-31' order by 访问次数 desc limit 10 """ - }, - { "current_date":"2023-08-31", + "sql":"""select 用户名 from 内容库产品 order by 访问次数 desc limit 10""" + }, + { "table_name":"艺人库", - "fields_list":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""", - "question":"近90天袁亚伟播放量平均值是多少", - "prior_schema_links":"""['152789226'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“近90天袁亚伟播放量平均值是多少“中，我们被问： + "fields_list":"""['歌手名', '歌手等级', '歌手类型', '歌手来源', '结算播放量', '运营播放量', '历史累计结算播放量', '有播放量歌曲数', '历史累计运营播放量', '付费用户结算播放量', '结算播放量占比', '运营播放份额', '完播量']""", + "question":"近90天袁呀味播放量平均值是多少", + "analysis": """让我们一步一步地思考。在问题“近90天袁呀味播放量平均值是多少“中，我们被问： “播放量平均值是多少”，所以我们需要column=[结算播放量] -”袁亚伟“，所以我们需要column=[歌手名] -”近90天“，所以我们需要column=[数据日期] -基于table和columns，可能的cell values 是 = ['袁亚伟', 90]。""", - "schema_links":"""["结算播放量", "歌手名", "数据日期", "'袁亚伟'", 90]""", - "sql":"""select avg(结算播放量) from 艺人库 where 歌手名 = '袁亚伟' and datediff('day', 数据日期, '2023-08-31') <= 90 """ - }, - { "current_date":"2023-08-31", +”袁呀味“，所以我们需要column=[歌手名] +基于table和columns，可能的cell values 是 = ['袁呀味']。""", + "schema_links":"""["结算播放量", "歌手名", "'袁呀味'"]""", + "sql":"""select avg(结算播放量) from 艺人库 where 歌手名 = '袁呀味'""" + }, + { "table_name":"艺人库", - "fields_list":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""", - "question":"周倩倩近7天结算播放量总和是多少", - "prior_schema_links":"""['199509'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“周倩倩近7天结算播放量总和是多少“中，我们被问： + "fields_list":"""['歌手名', '歌手等级', '歌手类型', '歌手来源', '结算播放量', '运营播放量', '历史累计结算播放量', '有播放量歌曲数', '历史累计运营播放量', '付费用户结算播放量', '结算播放量占比', '运营播放份额', '完播量']""", + "question":"周浅近7天结算播放量总和是多少", + "analysis": """让我们一步一步地思考。在问题“周浅近7天结算播放量总和是多少“中，我们被问： “结算播放量总和是多少”，所以我们需要column=[结算播放量] -”周倩倩“，所以我们需要column=[歌手名] -”近7天“，所以我们需要column=[数据日期] -基于table和columns，可能的cell values 是 = ['周倩倩', 7]。""", - "schema_links":"""["结算播放量", "歌手名", "数据日期", "'周倩倩'", 7]""", - "sql":"""select sum(结算播放量) from 艺人库 where 歌手名 = '周倩倩' and datediff('day', 数据日期, '2023-08-31') <= 7 """ - }, - { "current_date":"2023-09-14", +”周浅“，所以我们需要column=[歌手名] +基于table和columns，可能的cell values 是 = ['周浅']。""", + "schema_links":"""["结算播放量", "歌手名", "'周浅'"]""", + "sql":"""select sum(结算播放量) from 艺人库 where 歌手名 = '周浅'""" + }, + { "table_name":"内容库产品", - "fields_list":"""["部门", "模块", "用户名", "访问次数", "访问人数", "访问时长", "数据日期"]""", + "fields_list":"""['部门', '模块', '用户名', '访问次数', '访问人数', '访问时长']""", "question":"内容库访问次数大于1k的部门是哪些", - "prior_schema_links":"""[]""", "analysis": """让我们一步一步地思考。在问题“内容库访问次数大于1k的部门是哪些“中，我们被问： “部门是哪些”，所以我们需要column=[部门] ”访问次数大于1k的“，所以我们需要column=[访问次数] 基于table和columns，可能的cell values 是 = [1000]。""", "schema_links":"""["部门", "访问次数", 1000]""", - "sql":"""select 部门 from 内容库产品 where 访问次数 > 1000 and 数据日期 = '2023-09-14' """ - }, - { "current_date":"2023-09-18", + "sql":"""select 部门 from 内容库产品 where 访问次数 > 1000""" + }, + { "table_name":"歌曲库", - "fields_list":"""["歌曲名", "MPPM歌手ID", "歌曲版本", "歌曲类型", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"陈亿训唱的所有的播放量大于20k的孤勇者有哪些", - "prior_schema_links":"""['199509'->MPPM歌手ID, '1527123'->MPPM歌曲ID]""", - "analysis": """让我们一步一步地思考。在问题“陈亿训唱的所有的播放量大于20k的孤勇者有哪些“中，我们被问： -“孤勇者有哪些”，所以我们需要column=[歌曲名] + "fields_list":"""['歌曲ID', '歌曲MID', '歌曲名', '歌曲版本', '歌曲类型', '翻唱类型', '结算播放量', '运营播放量', '付费用户结算播放量', '历史累计结算播放量', '运营搜播量', '结算搜播量', '运营完播量', '运营推播量', '近7日复播率', '日均搜播量']""", + "question":"陈奕迅唱的所有的播放量大于20k的雇佣者有哪些", + "analysis": """让我们一步一步地思考。在问题“陈易迅唱的所有的播放量大于20k的雇佣者有哪些“中，我们被问： +“雇佣者有哪些”，所以我们需要column=[歌曲名] ”播放量大于20k的“，所以我们需要column=[结算播放量] -”陈亿训唱的“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = [20000, '陈亿训', '孤勇者']。""", - "schema_links":"""["歌曲名", "结算播放量", "歌手名", 20000, "'陈亿训'", "'孤勇者'"]""", - "sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 20000 and 歌手名 = '陈亿训' and 歌曲名 = '孤勇者' and 数据日期 = '2023-09-18' """ - }, - { "current_date":"2023-09-18", - "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"周洁轮去年发布的歌曲有哪些", - "prior_schema_links":"""['23109'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“周洁轮去年发布的歌曲有哪些“中，我们被问： -“歌曲有哪些”，所以我们需要column=[歌曲名] -”去年发布的“，所以我们需要column=[发布时间] -”周洁轮“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = ['周洁轮', 1]。""", - "schema_links":"""["歌曲名", "发布时间", "歌手名", 1, "'周洁轮'"]""", - "sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发布时间, '2023-09-18') <= 1 and 歌手名 = '周洁轮' and 数据日期 = '2023-09-18' """ - }, - { "current_date":"2023-09-11", - "table_name":"艺人库", - "fields_list":"""["播放量层级", "播放量单调性", "播放量方差", "播放量突增类型", "播放量集中度", "歌手名", "歌手等级", "歌手类型", "歌手来源", "签约日期", "MPPM潮流人等级", "结算播放量", "运营播放量", "历史累计结算播放量", "有播放量歌曲数", "历史累计运营播放量", "付费用户结算播放量", "结算播放量占比", "运营播放份额", "免费用户结算播放占比", "完播量", "数据日期"]""", - "question":"我想要近半年签约的播放量前十的歌手有哪些", - "prior_schema_links":"""[]""", - "analysis": """让我们一步一步地思考。在问题“我想要近半年签约的播放量前十的歌手“中，我们被问： -“歌手有哪些”，所以我们需要column=[歌手名] -”播放量前十的“，所以我们需要column=[结算播放量] -”近半年签约的“，所以我们需要column=[签约日期] -基于table和columns，可能的cell values 是 = [0.5, 10]。""", - "schema_links":"""["歌手名", "结算播放量", "签约日期", 0.5, 10]""", - "sql":"""select 歌手名 from 艺人库 where datediff('year', 签约日期, '2023-09-11') <= 0.5 and 数据日期 = '2023-09-11' order by 结算播放量 desc limit 10""" - }, - { "current_date":"2023-08-12", - "table_name":"歌曲库", - "fields_list": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""", - "question":"最近一年发行的歌曲中，有哪些在近7天播放超过一千万的", - "prior_schema_links":"""[]""", - "analysis": """让我们一步一步地思考。在问题“最近一年发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问： -“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名] -”最近一年发行的“，所以我们需要column=[发行日期] -”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量] -基于table和columns，可能的cell values 是 = [1, 10000000]""", - "schema_links":"""["歌曲名", "发行日期", "数据日期", "结算播放量", 1, 10000000]""", - "sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 1 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000""" - }, - { "current_date":"2023-08-12", - "table_name":"歌曲库", - "fields_list": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""", - "question":"今年以来发行的歌曲中，有哪些在近7天播放超过一千万的", - "prior_schema_links":"""[]""", - "analysis": """让我们一步一步地思考。在问题“今年以来发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问： -“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名] -”今年以来发行的“，所以我们需要column=[发行日期] -”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量] -基于table和columns，可能的cell values 是 = [0, 7, 10000000]""", - "schema_links":"""["歌曲名", "发行日期", "数据日期", "结算播放量", 0, 7, 10000000]""", - "sql":"""select 歌曲名 from 歌曲库 where datediff('year', 发行日期, '2023-08-12') <= 0 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000""" - }, - { "current_date":"2023-08-12", - "table_name":"歌曲库", - "fields_list": """["发行日期", "歌曲语言", "歌曲来源", "歌曲流派", "歌曲名", "歌曲版本", "歌曲类型", "发行时间", "数据日期"]""", - "question":"2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的", - "prior_schema_links":"""['514129144'->MPPM歌曲ID]""", - "analysis": """让我们一步一步地思考。在问题“2023年以来发行的歌曲中，有哪些在近7天播放超过一千万的“中，我们被问： -“发行的歌曲中，有哪些”，所以我们需要column=[歌曲名] -”2023年以来发行的“，所以我们需要column=[发行日期] -”在近7天播放超过一千万的“，所以我们需要column=[数据日期, 结算播放量] -基于table和columns，可能的cell values 是 = [2023, 7, 10000000]""", - "schema_links":"""["歌曲名", "发行日期", "数据日期", "结算播放量", 2023, 7, 10000000]""", - "sql":"""select 歌曲名 from 歌曲库 where YEAR(发行日期) >= 2023 and datediff('day', 数据日期, '2023-08-12') <= 7 and 结算播放量 > 10000000""" - }, - { "current_date":"2023-08-01", - "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"周洁轮2023年6月之后发布的歌曲有哪些", - "prior_schema_links":"""['23109'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“周洁轮2023年6月之后发布的歌曲有哪些“中，我们被问： -“歌曲有哪些”，所以我们需要column=[歌曲名] -”2023年6月之后发布的“，所以我们需要column=[发布时间] -”周洁轮“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = ['周洁轮', 2023, 6]。""", - "schema_links":"""["歌曲名", "发布时间", "歌手名", "周洁轮", 2023, 6]""", - "sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2023 and MONTH(发布时间) >= 6 and 歌手名 = '周洁轮' and 数据日期 = '2023-08-01' """ - }, - { "current_date":"2023-08-01", - "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？", - "prior_schema_links":"""['2312311'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“邓梓琦在2023年1月5日之后发布的歌曲中，有哪些播放量大于500W的？“中，我们被问： -“播放量大于500W的”，所以我们需要column=[结算播放量] -”邓梓琦在2023年1月5日之后发布的“，所以我们需要column=[发布时间] -”邓梓琦“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = ['邓梓琦', 2023, 1, 5, 5000000]。""", - "schema_links":"""["结算播放量", "发布时间", "歌手名", "邓梓琦", 2023, 1, 5, 5000000]""", - "sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2023 and MONTH(发布时间) >= 1 and DAY(发布时间) >= 5 and 歌手名 = '邓梓琦' and 结算播放量 > 5000000 and 数据日期 = '2023-08-01'""" - }, - { "current_date":"2023-09-17", - "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"2023年6月以后，张亮英播放量大于200万的歌曲有哪些？", - "prior_schema_links":"""['45453'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“2023年6月以后，张亮英播放量大于200万的歌曲有哪些？“中，我们被问： -“播放量大于200万的”，所以我们需要column=[结算播放量] -”2023年6月以后，张亮英“，所以我们需要column=[数据日期, 歌手名] -”歌曲有哪些“，所以我们需要column=[歌曲名] -基于table和columns，可能的cell values 是 = ['张亮英', 2023, 6, 2000000]。""", - "schema_links":"""["结算播放量", "数据日期", "歌手名", "张亮英", 2023, 6, 2000000]""", - "sql":"""select 歌曲名 from 歌曲库 where YEAR(数据日期) >= 2023 and MONTH(数据日期) >= 6 and 歌手名 = '张亮英' and 结算播放量 > 2000000 """ - }, - { "current_date":"2023-08-16", - "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些", - "prior_schema_links":"""['23109'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“2021年6月以后发布的李雨纯的播放量大于20万的歌曲有哪些“中，我们被问： -“播放量大于20万的”，所以我们需要column=[结算播放量] -”2021年6月以后发布的“，所以我们需要column=[发布时间] -”李雨纯“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = ['李雨纯', 2021, 6, 200000]。""", - "schema_links":"""["结算播放量", "发布时间", "歌手名", "李雨纯", 2021, 6, 200000]""", - "sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 2021 and MONTH(发布时间) >= 6 and 歌手名 = '李雨纯' and 结算播放量 > 200000 and 数据日期 = '2023-08-16'""" - }, - { "current_date":"2023-08-16", - "table_name":"歌曲库", - "fields_list":"""["歌曲名", "歌曲版本", "歌手名", "歌曲类型", "发布时间", "MPPM歌曲ID", "是否严选窄口径歌曲", "是否严选宽口径歌曲", "是否潮流人歌曲", "超声波歌曲ID", "C音歌曲ID", "C音歌曲MID", "结算播放量", "运营播放量", "分享量", "收藏量", "运营搜播量", "结算搜播量", "拉新用户数", "拉活用户数", "分享率", "结算播放份额", "数据日期"]""", - "question":"刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些", - "prior_schema_links":"""['4234234'->MPPM歌手ID]""", - "analysis": """让我们一步一步地思考。在问题“刘锝桦在1992年4月2日到2020年5月2日之间发布的播放量大于20万的歌曲有哪些“中，我们被问： -“播放量大于20万的”，所以我们需要column=[结算播放量] -”1992年4月2日到2020年5月2日之间发布的“，所以我们需要column=[发布时间] -”刘锝桦“，所以我们需要column=[歌手名] -基于table和columns，可能的cell values 是 = ['刘锝桦', 1992, 4, 2, 2020, 5, 2, 200000]。""", - "schema_links":"""["结算播放量", "发布时间", "歌手名", "刘锝桦", 1992, 4, 2, 2020, 5, 2, 200000]""", - "sql":"""select 歌曲名 from 歌曲库 where YEAR(发布时间) >= 1992 and MONTH(发布时间) >= 4 and DAY(发布时间) >= 2 and YEAR(发布时间) <= 2020 and MONTH(发布时间) <= 5 and DAY(发布时间) <= 2 and 歌手名 = '刘锝桦' and 结算播放量 > 200000 and 数据日期 = '2023-08-16'""" - } +”陈易迅唱的“，所以我们需要column=[歌手名] +基于table和columns，可能的cell values 是 = [20000, '陈易迅']。""", + "schema_links":"""["歌曲名", "结算播放量", "歌手名", 20000, "'陈易迅'"]""", + "sql":"""select 歌曲名 from 歌曲库 where 结算播放量 > 20000 and 歌手名 = '陈易迅'""" + } ] \ No newline at end of file diff --git a/chat/core/src/main/python/llm/preset_retrieval/run.py b/chat/core/src/main/python/llm/preset_retrieval/run.py index dc501b49c..9027253bf 100644 --- a/chat/core/src/main/python/llm/preset_retrieval/run.py +++ b/chat/core/src/main/python/llm/preset_retrieval/run.py @@ -8,7 +8,8 @@ from typing import Any, List, Mapping, Optional, Union sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) - +import chromadb +from chromadb.config import Settings from chromadb.api import Collection, Documents, Embeddings from langchain.llms import OpenAI @@ -20,9 +21,13 @@ from preset_query_db import (get_ids, add2preset_query_collection, from util.text2vec import Text2VecEmbeddingFunction from run_config import CHROMA_DB_PERSIST_PATH, PRESET_QUERY_COLLECTION_NAME -from util.chromadb_instance import client +client = chromadb.Client(Settings( + chroma_db_impl="duckdb+parquet", + persist_directory=CHROMA_DB_PERSIST_PATH # Optional, defaults to .chromadb/ in the current directory +)) + emb_func = Text2VecEmbeddingFunction() collection = client.get_or_create_collection(name=PRESET_QUERY_COLLECTION_NAME, @@ -30,8 +35,6 @@ collection = client.get_or_create_collection(name=PRESET_QUERY_COLLECTION_NAME, metadata={"hnsw:space": "cosine"} ) # Get a collection object from an existing collection, by name. If it doesn't exist, create it. -print("init_preset_query_collection_size: ", preset_query_collection_size(collection)) - def preset_query_retrieval_run(collection:Collection, query_texts_list:List[str], n_results:int=5): retrieval_res = query2preset_query_collection(collection=collection, diff --git a/chat/core/src/main/python/llm/run_config.py b/chat/core/src/main/python/llm/run_config.py index e2b47b404..989b44e5a 100644 --- a/chat/core/src/main/python/llm/run_config.py +++ b/chat/core/src/main/python/llm/run_config.py @@ -9,7 +9,6 @@ TEMPERATURE = 0.0 CHROMA_DB_PERSIST_DIR = 'chm_db' PRESET_QUERY_COLLECTION_NAME = "preset_query_collection" -TEXT2DSL_COLLECTION_NAME = "text2dsl_collection" CHROMA_DB_PERSIST_PATH = os.path.join(PROJECT_DIR_PATH, CHROMA_DB_PERSIST_DIR) diff --git a/chat/core/src/main/python/llm/sql/constructor.py b/chat/core/src/main/python/llm/sql/constructor.py deleted file mode 100644 index c6f367492..000000000 --- a/chat/core/src/main/python/llm/sql/constructor.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding:utf-8 -*- -import os -import sys - -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - -from langchain.prompts.few_shot import FewShotPromptTemplate -from langchain.prompts import PromptTemplate -from langchain.vectorstores import Chroma -from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings -from langchain.prompts.example_selector import SemanticSimilarityExampleSelector - -import chromadb -from chromadb.config import Settings - -from few_shot_example.sql_exampler import examplars as din_sql_examplars -from util.text2vec import Text2VecEmbeddingFunction, hg_embedding -from util.chromadb_instance import client as chromadb_client - - -from run_config import TEXT2DSL_COLLECTION_NAME - - -vectorstore = Chroma(collection_name=TEXT2DSL_COLLECTION_NAME, - embedding_function=hg_embedding, - client=chromadb_client) - -example_nums = 15 - -schema_linking_example_selector = SemanticSimilarityExampleSelector(vectorstore=vectorstore, k=example_nums, - input_keys=["question"], - example_keys=["table_name", "fields_list", "prior_schema_links", "question", "analysis", "schema_links"]) - -sql_example_selector = SemanticSimilarityExampleSelector(vectorstore=vectorstore, k=example_nums, - input_keys=["question"], - example_keys=["question", "current_date", "table_name", "schema_links", "sql"]) - -if vectorstore._collection.count() > 0: - print("examples already in din_sql_vectorstore") - print("init din_sql_vectorstore size:", vectorstore._collection.count()) - if vectorstore._collection.count() < len(din_sql_examplars): - print("din_sql_examplars size:", len(din_sql_examplars)) - vectorstore._collection.delete() - print("empty din_sql_vectorstore") - for example in din_sql_examplars: - schema_linking_example_selector.add_example(example) - print("added din_sql_vectorstore size:", vectorstore._collection.count()) -else: - for example in din_sql_examplars: - schema_linking_example_selector.add_example(example) - - print("added din_sql_vectorstore size:", vectorstore._collection.count()) diff --git a/chat/core/src/main/python/llm/sql/output_parser.py b/chat/core/src/main/python/llm/sql/output_parser.py index c90388850..64df5ba1f 100644 --- a/chat/core/src/main/python/llm/sql/output_parser.py +++ b/chat/core/src/main/python/llm/sql/output_parser.py @@ -1,13 +1,15 @@ # -*- coding:utf-8 -*- import re -def schema_link_parse(schema_link_output): - try: - schema_link_output = schema_link_output.strip() - pattern = r'Schema_links:(.*)' - schema_link_output = re.findall(pattern, schema_link_output, re.DOTALL)[0].strip() - except Exception as e: - print(e) - schema_link_output = None - return schema_link_output \ No newline at end of file +def schema_link_parse(schema_link_output): + try: + schema_link_output = schema_link_output.strip() + pattern = r'Schema_links:(.*)' + schema_link_output = re.findall(pattern, schema_link_output, re.DOTALL)[ + 0].strip() + except Exception as e: + print(e) + schema_link_output = None + + return schema_link_output diff --git a/chat/core/src/main/python/llm/sql/prompt_maker.py b/chat/core/src/main/python/llm/sql/prompt_maker.py index 0cfed83b1..6e05f95b3 100644 --- a/chat/core/src/main/python/llm/sql/prompt_maker.py +++ b/chat/core/src/main/python/llm/sql/prompt_maker.py @@ -1,5 +1,8 @@ # -*- coding:utf-8 -*- from typing import Any, List, Mapping, Optional, Union +import requests +import logging +import json import os import sys @@ -8,68 +11,78 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__))) from langchain.prompts import PromptTemplate from langchain.prompts.few_shot import FewShotPromptTemplate -from langchain.prompts.example_selector import SemanticSimilarityExampleSelector +from langchain.llms import OpenAI + +from few_shot_example.sql_exampler import examplars +from output_parser import schema_link_parse + + +def schema_linking_prompt_maker(user_query: str, model_name: str, + fields_list: List[str], + few_shots_example: str): + instruction = "# 根据数据库的表结构,找出为每个问题生成SQL查询语句的schema_links\n" + + schema_linking_prompt = "Table {table_name}, columns = {fields_list}\n问题:{user_query}\n分析: 让我们一步一步地思考。".format( + table_name=model_name, + fields_list=fields_list, + user_query=user_query) + + return instruction + few_shots_example + schema_linking_prompt def schema_linking_exampler(user_query: str, - domain_name: str, - fields_list: List[str], - prior_schema_links: Mapping[str,str], - example_selector: SemanticSimilarityExampleSelector, - ) -> str: + model_name: str, + fields_list: List[str] +) -> str: + example_prompt_template = PromptTemplate( + input_variables=["table_name", "fields_list", "question", "analysis", + "schema_links"], + template="Table {table_name}, columns = {fields_list}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schema_links}") - prior_schema_links_str = '['+ ','.join(["""'{}'->{}""".format(k,v) for k,v in prior_schema_links.items()]) + ']' + instruction = "# 根据数据库的表结构,找出为每个问题生成SQL查询语句的schema_links" - example_prompt_template = PromptTemplate(input_variables=["table_name", "fields_list", "prior_schema_links", "question", "analysis", "schema_links"], - template="Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\n问题:{question}\n分析:{analysis} 所以Schema_links是:\nSchema_links:{schema_links}") + schema_linking_prompt = "Table {table_name}, columns = {fields_list}\n问题:{question}\n分析: 让我们一步一步地思考。" - instruction = "# 根据数据库的表结构,参考先验信息,找出为每个问题生成SQL查询语句的schema_links" + schema_linking_example_prompt_template = FewShotPromptTemplate( + examples=examplars, + example_prompt=example_prompt_template, + example_separator="\n\n", + prefix=instruction, + input_variables=["table_name", "fields_list", "question"], + suffix=schema_linking_prompt + ) - schema_linking_prompt = "Table {table_name}, columns = {fields_list}, prior_schema_links = {prior_schema_links}\n问题:{question}\n分析: 让我们一步一步地思考。" + schema_linking_example_prompt = schema_linking_example_prompt_template.format( + table_name=model_name, + fields_list=fields_list, + question=user_query) - schema_linking_example_prompt_template = FewShotPromptTemplate( - example_selector=example_selector, - example_prompt=example_prompt_template, - example_separator="\n\n", - prefix=instruction, - input_variables=["table_name", "fields_list", "prior_schema_links", "question"], - suffix=schema_linking_prompt - ) - - schema_linking_example_prompt = schema_linking_example_prompt_template.format(table_name=domain_name, - fields_list=fields_list, - prior_schema_links=prior_schema_links_str, - question=user_query) - - return schema_linking_example_prompt + return schema_linking_example_prompt def sql_exampler(user_query: str, - domain_name: str, - schema_link_str: str, - data_date: str, - example_selector: SemanticSimilarityExampleSelector, - ) -> str: - - instruction = "# 根据schema_links为每个问题生成SQL查询语句" + model_name: str, + schema_link_str: str +) -> str: + instruction = "# 根据schema_links为每个问题生成SQL查询语句" - sql_example_prompt_template = PromptTemplate(input_variables=["question", "current_date", "table_name", "schema_links", "sql"], - template="问题:{question}\nCurrent_date:{current_date}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:{sql}") + sql_example_prompt_template = PromptTemplate( + input_variables=["question", "table_name", "schema_links", "sql"], + template="问题:{question}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:{sql}") - sql_prompt = "问题:{question}\nCurrent_date:{current_date}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:" + sql_prompt = "问题:{question}\nTable {table_name}\nSchema_links:{schema_links}\nSQL:" - sql_example_prompt_template = FewShotPromptTemplate( - example_selector=example_selector, - example_prompt=sql_example_prompt_template, - example_separator="\n\n", - prefix=instruction, - input_variables=["question", "current_date", "table_name", "schema_links"], - suffix=sql_prompt - ) + sql_example_prompt_template = FewShotPromptTemplate( + examples=examplars, + example_prompt=sql_example_prompt_template, + example_separator="\n\n", + prefix=instruction, + input_variables=["question", "table_name", "schema_links"], + suffix=sql_prompt + ) - sql_example_prompt = sql_example_prompt_template.format(question=user_query, - current_date=data_date, - table_name=domain_name, - schema_links=schema_link_str) + sql_example_prompt = sql_example_prompt_template.format(question=user_query, + table_name=model_name, + schema_links=schema_link_str) - return sql_example_prompt + return sql_example_prompt diff --git a/chat/core/src/main/python/llm/sql/run.py b/chat/core/src/main/python/llm/sql/run.py index 34919799b..ea60d7f36 100644 --- a/chat/core/src/main/python/llm/sql/run.py +++ b/chat/core/src/main/python/llm/sql/run.py @@ -1,4 +1,6 @@ -from typing import List, Union, Mapping +# -*- coding:utf-8 -*- + +from typing import List, Union import logging import json import os @@ -7,54 +9,33 @@ import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from sql.prompt_maker import schema_linking_exampler, sql_exampler -from sql.constructor import schema_linking_example_selector, sql_example_selector -from sql.output_parser import schema_link_parse +from sql.prompt_maker import schema_linking_exampler, schema_link_parse, \ + sql_exampler from util.llm_instance import llm +def query2sql(query_text: str, schema: dict): + print("schema: ", schema) -def query2sql(query_text: str, - schema : Union[dict, None] = None, - current_date: str = None, - linking: Union[List[Mapping[str, str]], None] = None - ): - - print("query_text: ", query_text) - print("schema: ", schema) - print("current_date: ", current_date) - print("prior_schema_links: ", linking) + model_name = schema['modelName'] + fields_list = schema['fieldNameList'] - if linking is not None: - prior_schema_links = {item['fieldValue']:item['fieldName'] for item in linking} - else: - prior_schema_links = {} + schema_linking_prompt = schema_linking_exampler(query_text, model_name, + fields_list) + schema_link_output = llm(schema_linking_prompt) + schema_link_str = schema_link_parse(schema_link_output) - model_name = schema['modelName'] - fields_list = schema['fieldNameList'] + sql_prompt = sql_exampler(query_text, model_name, schema_link_str) + sql_output = llm(sql_prompt) - schema_linking_prompt = schema_linking_exampler(query_text, model_name, fields_list, prior_schema_links, schema_linking_example_selector) - print("schema_linking_prompt->", schema_linking_prompt) - schema_link_output = llm(schema_linking_prompt) - schema_link_str = schema_link_parse(schema_link_output) - - sql_prompt = sql_exampler(query_text, model_name, schema_link_str, current_date, sql_example_selector) - print("sql_prompt->", sql_prompt) - sql_output = llm(sql_prompt) + resp = dict() + resp['query'] = query_text + resp['model'] = model_name + resp['fields'] = fields_list - resp = dict() - resp['query'] = query_text - resp['model'] = model_name - resp['fields'] = fields_list - resp['priorSchemaLinking'] = linking - resp['dataDate'] = current_date + resp['schemaLinkingOutput'] = schema_link_output + resp['schemaLinkStr'] = schema_link_str - resp['schemaLinkingOutput'] = schema_link_output - resp['schemaLinkStr'] = schema_link_str - - resp['sqlOutput'] = sql_output - - print("resp: ", resp) - - return resp + resp['sqlOutput'] = sql_output + return resp diff --git a/chat/core/src/main/python/llm/util/chromadb_instance.py b/chat/core/src/main/python/llm/util/chromadb_instance.py deleted file mode 100644 index f0fe6ce01..000000000 --- a/chat/core/src/main/python/llm/util/chromadb_instance.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding:utf-8 -*- -import chromadb -from chromadb.config import Settings - -from run_config import CHROMA_DB_PERSIST_PATH - -client = chromadb.Client(Settings( - chroma_db_impl="duckdb+parquet", - persist_directory=CHROMA_DB_PERSIST_PATH # Optional, defaults to .chromadb/ in the current directory -)) \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/DateFieldCorrectorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/DateFieldCorrectorTest.java index 136aabec4..4bc2a3919 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/DateFieldCorrectorTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/DateFieldCorrectorTest.java @@ -1,37 +1,45 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import static org.mockito.ArgumentMatchers.any; + import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper; import org.junit.Assert; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; class DateFieldCorrectorTest { @Test - void rewriter() { + void corrector() { + MockedStatic dslDateHelper = Mockito.mockStatic(DSLDateHelper.class); + + dslDateHelper.when(() -> DSLDateHelper.getReferenceDate(any())).thenReturn("2023-08-14"); DateFieldCorrector dateFieldCorrector = new DateFieldCorrector(); SemanticParseInfo parseInfo = new SemanticParseInfo(); SchemaElement model = new SchemaElement(); model.setId(2L); parseInfo.setModel(model); - CorrectionInfo correctionInfo = CorrectionInfo.builder() + SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder() .sql("select count(歌曲名) from 歌曲库 ") .parseInfo(parseInfo) .build(); - CorrectionInfo rewriter = dateFieldCorrector.corrector(correctionInfo); + dateFieldCorrector.correct(semanticCorrectInfo); - Assert.assertEquals("SELECT count(歌曲名) FROM 歌曲库 WHERE 数据日期 = '2023-08-14'", rewriter.getSql()); + Assert.assertEquals("SELECT count(歌曲名) FROM 歌曲库 WHERE 数据日期 = '2023-08-14'", semanticCorrectInfo.getSql()); - correctionInfo = CorrectionInfo.builder() + semanticCorrectInfo = SemanticCorrectInfo.builder() .sql("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'") .parseInfo(parseInfo) .build(); - rewriter = dateFieldCorrector.corrector(correctionInfo); + dateFieldCorrector.correct(semanticCorrectInfo); - Assert.assertEquals("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'", rewriter.getSql()); + Assert.assertEquals("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'", semanticCorrectInfo.getSql()); } } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldNameCorrectorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldNameCorrectorTest.java index b9e6351b3..7caae3c06 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldNameCorrectorTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldNameCorrectorTest.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; @@ -16,10 +16,10 @@ import org.junit.jupiter.api.Test; class FieldNameCorrectorTest { @Test - void rewriter() { + void corrector() { FieldNameCorrector corrector = new FieldNameCorrector(); - CorrectionInfo correctionInfo = CorrectionInfo.builder() + SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder() .sql("select 歌曲名 from 歌曲库 where 专辑照片 = '七里香' and 专辑名 = '流行' and 数据日期 = '2023-08-19'") .build(); @@ -55,11 +55,11 @@ class FieldNameCorrectorTest { properties.put(Constants.CONTEXT, dslParseResult); parseInfo.setProperties(properties); - correctionInfo.setParseInfo(parseInfo); + semanticCorrectInfo.setParseInfo(parseInfo); - CorrectionInfo rewriter = corrector.corrector(correctionInfo); + corrector.correct(semanticCorrectInfo); Assert.assertEquals("SELECT 歌曲名 FROM 歌曲库 WHERE 歌曲名 = '七里香' AND 歌曲流派 = '流行' AND 数据日期 = '2023-08-19'", - rewriter.getSql()); + semanticCorrectInfo.getSql()); } } \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldValueCorrectorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldValueCorrectorTest.java index 1cc8af14b..d9afccf23 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldValueCorrectorTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldValueCorrectorTest.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.chat.corrector; import static org.mockito.Mockito.when; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.common.util.ContextUtils; @@ -53,19 +53,19 @@ class FieldValueCorrectorTest { SchemaElement model = new SchemaElement(); model.setId(2L); parseInfo.setModel(model); - CorrectionInfo correctionInfo = CorrectionInfo.builder() + SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder() .sql("select count(song_name) from 歌曲库 where singer_name = '周先生'") .parseInfo(parseInfo) .build(); FieldValueCorrector corrector = new FieldValueCorrector(); - CorrectionInfo info = corrector.corrector(correctionInfo); + corrector.correct(semanticCorrectInfo); - Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", info.getSql()); + Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", semanticCorrectInfo.getSql()); - correctionInfo.setSql("select count(song_name) from 歌曲库 where singer_name = '杰伦'"); - info = corrector.corrector(correctionInfo); + semanticCorrectInfo.setSql("select count(song_name) from 歌曲库 where singer_name = '杰伦'"); + corrector.correct(semanticCorrectInfo); - Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", info.getSql()); + Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", semanticCorrectInfo.getSql()); } } \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrectorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrectorTest.java index 99da9af35..13d648a34 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrectorTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrectorTest.java @@ -1,25 +1,26 @@ package com.tencent.supersonic.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import org.junit.Assert; import org.junit.jupiter.api.Test; class SelectFieldAppendCorrectorTest { @Test - void rewriter() { + void corrector() { SelectFieldAppendCorrector corrector = new SelectFieldAppendCorrector(); - CorrectionInfo correctionInfo = CorrectionInfo.builder() + SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder() .sql("select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' " + "and sys_imp_date = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11") .build(); - CorrectionInfo rewriter = corrector.corrector(correctionInfo); + corrector.correct(semanticCorrectInfo); Assert.assertEquals( - "SELECT 歌曲名, 歌手名, 歌曲发布时, 发布日期 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " - + "AND 歌手名 = '邓紫棋' AND sys_imp_date = '2023-08-09' " - + "AND 歌曲发布时 = '2023-08-01' ORDER BY 播放量 DESC LIMIT 11", rewriter.getSql()); + "SELECT 歌曲名, 歌手名, 播放量, 歌曲发布时, 发布日期 FROM 歌曲库 WHERE " + + "datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '邓紫棋' " + + "AND sys_imp_date = '2023-08-09' AND 歌曲发布时 = '2023-08-01'" + + " ORDER BY 播放量 DESC LIMIT 11", semanticCorrectInfo.getSql()); } } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java index 39056445e..33e86c177 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.chat.parser.llm.dsl; import static org.mockito.Mockito.when; -import com.tencent.supersonic.chat.api.pojo.CorrectionInfo; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.common.util.ContextUtils; @@ -67,14 +67,14 @@ class LLMDslParserTest { SchemaElement model = new SchemaElement(); model.setId(2L); parseInfo.setModel(model); - CorrectionInfo correctionInfo = CorrectionInfo.builder() + SemanticCorrectInfo semanticCorrectInfo = SemanticCorrectInfo.builder() .sql("select count(song_name) from 歌曲库 where singer_name = '周先生' and YEAR(publish_time) >= 2023 and ") .parseInfo(parseInfo) .build(); LLMDslParser llmDslParser = new LLMDslParser(); - llmDslParser.setFilter(correctionInfo, 2L, parseInfo); + llmDslParser.setFilter(semanticCorrectInfo, 2L, parseInfo); } } \ No newline at end of file diff --git a/chat/knowledge/pom.xml b/chat/knowledge/pom.xml index 5fed94b0e..102b7289f 100644 --- a/chat/knowledge/pom.xml +++ b/chat/knowledge/pom.xml @@ -11,10 +11,6 @@ chat-knowledge - - com.google.guava - guava - org.springframework spring-context @@ -111,8 +107,9 @@ com.tencent.supersonic semantic-query ${project.version} + compile - \ No newline at end of file + diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java index d2b732ca8..67f041230 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java @@ -375,4 +375,4 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { return true; } } -} \ No newline at end of file +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java index a600b972c..219c4d072 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java @@ -5,7 +5,7 @@ import java.util.List; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; /** diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DimensionWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DimensionWordBuilder.java index c991d89e4..8ea7eeee8 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DimensionWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DimensionWordBuilder.java @@ -7,7 +7,7 @@ import java.util.List; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DomainWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DomainWordBuilder.java index 554437972..a7628d776 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DomainWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/DomainWordBuilder.java @@ -6,7 +6,7 @@ import java.util.List; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java index d2b30b635..f7f2e5ad1 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java @@ -7,7 +7,7 @@ import java.util.Objects; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/MetricWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/MetricWordBuilder.java index d51c3190a..d4e1827d5 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/MetricWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/MetricWordBuilder.java @@ -7,7 +7,7 @@ import java.util.List; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/ValueWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/ValueWordBuilder.java index bc0a24462..109b72b93 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/ValueWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/ValueWordBuilder.java @@ -7,7 +7,7 @@ import java.util.Objects; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/WordBuilderFactory.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/WordBuilderFactory.java index e940a7fc5..527e9075e 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/WordBuilderFactory.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/WordBuilderFactory.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.knowledge.dictionary.builder; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/ApplicationStartedInit.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java similarity index 87% rename from chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/ApplicationStartedInit.java rename to chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java index 5deb2dda1..a9f1e8161 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/ApplicationStartedInit.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.knowledge; +package com.tencent.supersonic.knowledge.listener; import com.tencent.supersonic.knowledge.dictionary.DictWord; import com.tencent.supersonic.knowledge.service.SchemaService; @@ -16,7 +16,7 @@ import java.util.List; @Slf4j @Component -public class ApplicationStartedInit implements ApplicationListener { +public class ApplicationStartedListener implements ApplicationListener { @Autowired private KnowledgeService knowledgeService; @@ -27,6 +27,11 @@ public class ApplicationStartedInit implements ApplicationListener { + @Override + public void onApplicationEvent(DataAddEvent event) { + DictWord dictWord = new DictWord(); + dictWord.setWord(event.getName()); + String sign = DictWordType.NATURE_SPILT; + String nature = sign + event.getModelId() + sign + event.getId() + event.getType(); + String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; + dictWord.setNature(nature); + dictWord.setNatureWithFrequency(natureWithFrequency); + log.info("dataAddListener begins to add data:{}", dictWord); + HanlpHelper.addToCustomDictionary(dictWord); + } +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java new file mode 100644 index 000000000..9d812e348 --- /dev/null +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java @@ -0,0 +1,27 @@ +package com.tencent.supersonic.knowledge.listener; + +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DataDeleteEvent; +import com.tencent.supersonic.knowledge.dictionary.DictWord; +import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.knowledge.utils.HanlpHelper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class DataDeleteListener implements ApplicationListener { + @Override + public void onApplicationEvent(DataDeleteEvent event) { + DictWord dictWord = new DictWord(); + dictWord.setWord(event.getName()); + String sign = DictWordType.NATURE_SPILT; + String nature = sign + event.getModelId() + sign + event.getId() + event.getType(); + String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; + dictWord.setNature(nature); + dictWord.setNatureWithFrequency(natureWithFrequency); + log.info("dataDeleteListener begins to delete data:{}", dictWord); + HanlpHelper.removeFromCustomDictionary(dictWord); + } +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java new file mode 100644 index 000000000..13db47355 --- /dev/null +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java @@ -0,0 +1,29 @@ +package com.tencent.supersonic.knowledge.listener; + +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DataUpdateEvent; +import com.tencent.supersonic.knowledge.dictionary.DictWord; +import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.knowledge.utils.HanlpHelper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class DataUpdateListener implements ApplicationListener { + @Override + public void onApplicationEvent(DataUpdateEvent event) { + DictWord dictWord = new DictWord(); + dictWord.setWord(event.getName()); + String sign = DictWordType.NATURE_SPILT; + String nature = sign + event.getModelId() + sign + event.getId() + event.getType(); + String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; + dictWord.setNature(nature); + dictWord.setNatureWithFrequency(natureWithFrequency); + log.info("dataUpdateListener begins to update data:{}", dictWord); + HanlpHelper.removeFromCustomDictionary(dictWord); + dictWord.setWord(event.getNewName()); + HanlpHelper.addToCustomDictionary(dictWord); + } +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/KnowledgeServiceImpl.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/KnowledgeServiceImpl.java index 1f089a5d4..5f42a5823 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/KnowledgeServiceImpl.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/KnowledgeServiceImpl.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.knowledge.service; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.utils.HanlpHelper; import java.util.List; @@ -52,4 +52,4 @@ public class KnowledgeServiceImpl implements KnowledgeService { } } -} \ No newline at end of file +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java index 49c6c2159..75dd6f66e 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java @@ -5,7 +5,7 @@ import com.hankcs.hanlp.collection.trie.bintrie.BinTrie; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.DictionaryAttributeUtil; import com.tencent.supersonic.knowledge.dictionary.MapResult; import java.util.Arrays; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/WordService.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/WordService.java index 5afff44b6..86ec65b73 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/WordService.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/WordService.java @@ -4,7 +4,7 @@ import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory; import java.util.ArrayList; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/HanlpHelper.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/HanlpHelper.java index 89987b103..e2b4a9420 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/HanlpHelper.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/HanlpHelper.java @@ -7,7 +7,7 @@ import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.seg.Segment; import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.DictWord; import java.io.FileNotFoundException; import java.io.IOException; @@ -20,6 +20,7 @@ import com.tencent.supersonic.knowledge.dictionary.HadoopFileIOAdapter; import com.tencent.supersonic.knowledge.service.SearchService; import com.tencent.supersonic.knowledge.dictionary.MultiCustomDictionary; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; import org.springframework.util.ResourceUtils; @@ -163,6 +164,29 @@ public class HanlpHelper { return getDynamicCustomDictionary().insert(dictWord.getWord(), dictWord.getNatureWithFrequency()); } + public static void removeFromCustomDictionary(DictWord dictWord) { + log.info("dictWord:{}", dictWord); + CoreDictionary.Attribute attribute = getDynamicCustomDictionary().get(dictWord.getWord()); + if (attribute != null) { + return; + } + log.info("get attribute:{}", attribute); + getDynamicCustomDictionary().remove(dictWord.getWord()); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < attribute.nature.length; i++) { + if (!attribute.nature[i].toString().equals(dictWord.getNature())) { + sb.append(attribute.nature[i].toString() + " "); + sb.append(attribute.frequency[i] + " "); + } + } + String natureWithFrequency = sb.toString(); + int len = natureWithFrequency.length(); + log.info("filtered natureWithFrequency:{}", natureWithFrequency); + if (StringUtils.isNotBlank(natureWithFrequency)) { + getDynamicCustomDictionary().add(dictWord.getWord(), natureWithFrequency.substring(0, len - 1)); + } + } + public static void transLetterOriginal(List mapResults) { if (CollectionUtils.isEmpty(mapResults)) { return; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java index 4dd6db8d5..0724108d2 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.knowledge.utils; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.knowledge.dictionary.DictWordType; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.ModelInfoStat; import java.util.ArrayList; import java.util.Comparator; diff --git a/common/pom.xml b/common/pom.xml index e515542ba..c02715ba1 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -89,17 +89,6 @@ ${yaml.utils.version} - - com.google.guava - guava - ${guava.version} - - - - org.springframework.boot - spring-boot-starter-data-redis - - com.github.dozermapper dozer-core diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java index 3dc422c3e..79fb6d1d0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java @@ -62,4 +62,6 @@ public class Constants { public static final String BRACKETS_START = "["; public static final String BRACKETS_END = "]"; + public static final Long DEFAULT_FREQUENCY = 100000L; + } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java new file mode 100644 index 000000000..07d100c8a --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java @@ -0,0 +1,51 @@ +package com.tencent.supersonic.common.pojo; + +import org.springframework.context.ApplicationEvent; + +public class DataAddEvent extends ApplicationEvent { + private static final long serialVersionUID = 1L; + private String name; + private Long modelId; + private Long id; + private String type; + + public DataAddEvent(Object source, String name, Long modelId, Long id, String type) { + super(source); + this.name = name; + this.modelId = modelId; + this.id = id; + this.type = type; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public Long getModelId() { + return modelId; + } + + public void setModelId(Long modelId) { + this.modelId = modelId; + } + + public void setType(String type) { + this.type = type; + } + + public String getType() { + return type; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java new file mode 100644 index 000000000..83bf309fe --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java @@ -0,0 +1,51 @@ +package com.tencent.supersonic.common.pojo; + +import org.springframework.context.ApplicationEvent; + +public class DataDeleteEvent extends ApplicationEvent { + private static final long serialVersionUID = 1L; + private String name; + private Long modelId; + private Long id; + private String type; + + public DataDeleteEvent(Object source, String name, Long modelId, Long id, String type) { + super(source); + this.name = name; + this.modelId = modelId; + this.id = id; + this.type = type; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public Long getModelId() { + return modelId; + } + + public void setModelId(Long modelId) { + this.modelId = modelId; + } + + public void setType(String type) { + this.type = type; + } + + public String getType() { + return type; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java new file mode 100644 index 000000000..908c28f09 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java @@ -0,0 +1,61 @@ +package com.tencent.supersonic.common.pojo; + +import org.springframework.context.ApplicationEvent; + +public class DataUpdateEvent extends ApplicationEvent { + private static final long serialVersionUID = 1L; + private String name; + private String newName; + private Long modelId; + private Long id; + private String type; + + public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, String type) { + super(source); + this.name = name; + this.newName = newName; + this.modelId = modelId; + this.id = id; + this.type = type; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public String getNewName() { + return newName; + } + + public void setNewName(String newName) { + this.newName = newName; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public Long getModelId() { + return modelId; + } + + public void setModelId(Long modelId) { + this.modelId = modelId; + } + + public void setType(String type) { + this.type = type; + } + + public String getType() { + return type; + } +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictWordType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java similarity index 96% rename from chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictWordType.java rename to common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java index 8f40ed4c0..66b3a0d13 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictWordType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.knowledge.dictionary; +package com.tencent.supersonic.common.pojo.enums; import org.apache.commons.lang3.StringUtils; diff --git a/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java index 4a5848957..939b2a04a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java @@ -83,8 +83,8 @@ public class ChatGptHelper { String table, String desc, Boolean isPercentage) { - String msg = "Assuming you are a professional data analyst specializing in indicators, " - + "you have a vast amount of data analysis indicator content. You are familiar with the basic" + String msg = "Assuming you are a professional data analyst specializing in metrics and dimensions, " + + "you have a vast amount of data analysis metrics content. You are familiar with the basic" + " format of the content,Now, Construct your answer Based on the following json-schema.\n" + "{\n" + "\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" @@ -104,10 +104,15 @@ public class ChatGptHelper { + mockType + " calculates the field source: " + bizName - + ", The description of this indicator is: " + + ", The description of this metrics is: " + desc - + ", provide some aliases for this，please take chinese or english," - + "but more chinese and Not repeating，\"\n" + + ", provide some aliases for this, please take chinese or english," + + "You must adhere to the following rules:\n" + + "1. Please do not generate aliases like xxx1, xxx2, xxx3.\n" + + "2. Please do not generate aliases that are the same as the original names of metrics/dimensions.\n" + + "3. Please pay attention to the quality of the generated aliases and " + + " avoid creating aliases that look like test data.\n" + + "4. Please generate more Chinese aliases." + "},\n" + "\"additionalProperties\":false}\n" + "Please double-check whether the answer conforms to the format described in the JSON-schema.\n" diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java index b743fc450..dbf0ffe0c 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.util; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; @@ -13,11 +14,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class DateUtils { - public static final String DATE_FORMAT_DOT = "yyyy-MM-dd"; + public static final String DATE_FORMAT = "yyyy-MM-dd"; + + public static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static Integer currentYear() { Date date = new Date(); - SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT); + SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); String time = dateFormat.format(date).replaceAll("-", ""); int year = Integer.parseInt(time.substring(0, 4)); return year; @@ -56,12 +59,19 @@ public class DateUtils { Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); calendar.add(Calendar.DAY_OF_MONTH, -intervalDay); - SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT); + SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); return dateFormat.format(calendar.getTime()); } + + public static String getBeforeDate(int intervalDay, DatePeriodEnum datePeriodEnum) { + SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + String currentDate = dateFormat.format(new Date()); + return getBeforeDate(currentDate, intervalDay, datePeriodEnum); + } + public static String getBeforeDate(String date, int intervalDay, DatePeriodEnum datePeriodEnum) { - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT_DOT); + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT); LocalDate currentDate = LocalDate.parse(date, dateTimeFormatter); LocalDate result = null; switch (datePeriodEnum) { @@ -89,8 +99,26 @@ public class DateUtils { default: } if (Objects.nonNull(result)) { - return result.format(DateTimeFormatter.ofPattern(DATE_FORMAT_DOT)); + return result.format(DateTimeFormatter.ofPattern(DATE_FORMAT)); } return null; } + + + public static String format(Date date) { + DateFormat dateFormat; + if (containsTime(date)) { + dateFormat = new SimpleDateFormat(DateUtils.TIME_FORMAT); + } else { + dateFormat = new SimpleDateFormat(DateUtils.DATE_FORMAT); + } + return dateFormat.format(date); + } + + private static boolean containsTime(Date date) { + DateFormat timeFormat = new SimpleDateFormat("HH:mm:ss"); + String timeString = timeFormat.format(date); + return !timeString.equals("00:00:00"); + } + } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java b/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java index 7781f224e..4add68aa9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java @@ -20,7 +20,7 @@ public class CaffeineCacheConfig { @Value("${caffeine.max.size:5000}") private Integer caffeineMaximumSize; - @Bean + @Bean(name = "caffeineCache") public Cache caffeineCache() { return Caffeine.newBuilder() .expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES) @@ -30,4 +30,15 @@ public class CaffeineCacheConfig { .maximumSize(caffeineMaximumSize) .build(); } -} \ No newline at end of file + + @Bean(name = "searchCaffeineCache") + public Cache searchCaffeineCache() { + return Caffeine.newBuilder() + .expireAfterWrite(10000, TimeUnit.MINUTES) + // 初始的缓存空间大小 + .initialCapacity(caffeineInitialCapacity) + // 缓存的最大条数 + .maximumSize(caffeineMaximumSize) + .build(); + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java b/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java index aa967f6f3..ebc41b60a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java @@ -6,6 +6,7 @@ import com.google.common.base.Joiner; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @Component @@ -15,7 +16,9 @@ public class CaffeineCacheImpl implements CacheUtils { @Autowired private CacheCommonConfig cacheCommonConfig; + @Autowired + @Qualifier("caffeineCache") private Cache caffeineCache; @Override @@ -46,4 +49,4 @@ public class CaffeineCacheImpl implements CacheUtils { caffeineCache.asMap().remove(key); return true; } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java index 0d592ffe1..78869857b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java @@ -16,6 +16,7 @@ import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; import net.sf.jsqlparser.expression.operators.relational.EqualsTo; import net.sf.jsqlparser.expression.operators.relational.GreaterThan; import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; +import net.sf.jsqlparser.expression.operators.relational.LikeExpression; import net.sf.jsqlparser.expression.operators.relational.MinorThan; import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; import net.sf.jsqlparser.schema.Column; @@ -29,6 +30,21 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter { this.filterExpressions = filterExpressions; } + public void visit(LikeExpression expr) { + Expression leftExpression = expr.getLeftExpression(); + Expression rightExpression = expr.getRightExpression(); + + FilterExpression filterExpression = new FilterExpression(); + String columnName = null; + if (leftExpression instanceof Column) { + Column column = (Column) leftExpression; + columnName = column.getColumnName(); + filterExpression.setFieldName(columnName); + } + filterExpression.setFieldValue(getFieldValue(rightExpression)); + filterExpression.setOperator(expr.getStringExpression()); + filterExpressions.add(filterExpression); + } @Override public void visit(MinorThan expr) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionNameReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionNameReplaceVisitor.java new file mode 100644 index 000000000..76fc5bd25 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionNameReplaceVisitor.java @@ -0,0 +1,25 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.Function; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class FunctionNameReplaceVisitor extends ExpressionVisitorAdapter { + + private Map functionMap; + + public FunctionNameReplaceVisitor(Map functionMap) { + this.functionMap = functionMap; + } + + public void visit(Function function) { + String replaceFunctionName = functionMap.get(function.getName().toLowerCase()); + if (StringUtils.isNotBlank(replaceFunctionName)) { + function.setName(replaceFunctionName); + } + } + +} \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByFunctionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByFunctionReplaceVisitor.java new file mode 100644 index 000000000..769fcb18d --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByFunctionReplaceVisitor.java @@ -0,0 +1,38 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.GroupByVisitor; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class GroupByFunctionReplaceVisitor implements GroupByVisitor { + + private Map functionMap; + + public GroupByFunctionReplaceVisitor(Map functionMap) { + this.functionMap = functionMap; + } + + public void visit(GroupByElement groupByElement) { + groupByElement.getGroupByExpressionList(); + ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList(); + List groupByExpressions = groupByExpressionList.getExpressions(); + + for (int i = 0; i < groupByExpressions.size(); i++) { + Expression expression = groupByExpressions.get(i); + if (expression instanceof Function) { + Function function = (Function) expression; + String replaceName = functionMap.get(function.getName().toLowerCase()); + if (StringUtils.isNotBlank(replaceName)) { + function.setName(replaceName); + } + } + } + } +} \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByVisitor.java new file mode 100644 index 000000000..9bce2c0d6 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByVisitor.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.statement.select.GroupByElement; + +@Slf4j +public class GroupByVisitor implements net.sf.jsqlparser.statement.select.GroupByVisitor { + + private boolean hasAggregateFunction = false; + + public void visit(GroupByElement groupByElement) { + this.hasAggregateFunction = true; + } + + public boolean isHasAggregateFunction() { + return hasAggregateFunction; + } +} \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java index c840207db..773110861 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.common.util.jsqlparser; +import java.util.List; import java.util.Set; import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.OrderByElement; import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter; @@ -20,6 +22,15 @@ public class OrderByAcquireVisitor extends OrderByVisitorAdapter { if (expression instanceof Column) { fields.add(((Column) expression).getColumnName()); } + if (expression instanceof Function) { + Function function = (Function) expression; + List expressions = function.getParameters().getExpressions(); + for (Expression column : expressions) { + if (column instanceof Column) { + fields.add(((Column) column).getColumnName()); + } + } + } super.visit(orderBy); } } \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByReplaceVisitor.java index 73636cad7..a3a4ccda6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByReplaceVisitor.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.common.util.jsqlparser; +import java.util.List; import java.util.Map; import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.OrderByElement; import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter; @@ -21,6 +23,15 @@ public class OrderByReplaceVisitor extends OrderByVisitorAdapter { if (expression instanceof Column) { parseVisitorHelper.replaceColumn((Column) expression, fieldToBizName); } + if (expression instanceof Function) { + Function function = (Function) expression; + List expressions = function.getParameters().getExpressions(); + for (Expression column : expressions) { + if (column instanceof Column) { + parseVisitorHelper.replaceColumn((Column) column, fieldToBizName); + } + } + } super.visit(orderBy); } } \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java index d7e9a0df8..ac3f6a6d6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java @@ -26,6 +26,7 @@ import org.springframework.util.CollectionUtils; */ @Slf4j public class SqlParserSelectHelper { + public static List getFilterExpression(String sql) { PlainSelect plainSelect = getPlainSelect(sql); if (Objects.isNull(plainSelect)) { @@ -132,16 +133,12 @@ public class SqlParserSelectHelper { } } List orderByElements = plainSelect.getOrderByElements(); - if (orderByElements != null) { + if (!CollectionUtils.isEmpty(orderByElements)) { for (OrderByElement orderByElement : orderByElements) { - Expression expression = orderByElement.getExpression(); - - if (expression instanceof Column) { - Column column = (Column) expression; - result.add(column.getColumnName()); - } + orderByElement.accept(new OrderByAcquireVisitor(result)); } } + Expression where = plainSelect.getWhere(); if (where != null) { where.accept(new ExpressionVisitorAdapter() { @@ -181,7 +178,17 @@ public class SqlParserSelectHelper { for (SelectItem selectItem : selectItems) { selectItem.accept(visitor); } - return visitor.hasAggregateFunction(); + boolean selectFunction = visitor.hasAggregateFunction(); + if (selectFunction) { + return true; + } + GroupByElement groupBy = plainSelect.getGroupBy(); + if (Objects.nonNull(groupBy)) { + GroupByVisitor replaceVisitor = new GroupByVisitor(); + groupBy.accept(replaceVisitor); + return replaceVisitor.isHasAggregateFunction(); + } + return false; } } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java index 044f08513..e853864cf 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java @@ -94,6 +94,31 @@ public class SqlParserUpdateHelper { return selectStatement.toString(); } + public static String replaceFunction(String sql, Map functionMap) { + Select selectStatement = SqlParserSelectHelper.getSelect(sql); + SelectBody selectBody = selectStatement.getSelectBody(); + if (!(selectBody instanceof PlainSelect)) { + return sql; + } + PlainSelect plainSelect = (PlainSelect) selectBody; + //1. replace where dataDiff function + Expression where = plainSelect.getWhere(); + + FunctionNameReplaceVisitor visitor = new FunctionNameReplaceVisitor(functionMap); + if (Objects.nonNull(where)) { + where.accept(visitor); + } + GroupByElement groupBy = plainSelect.getGroupBy(); + if (Objects.nonNull(groupBy)) { + GroupByFunctionReplaceVisitor replaceVisitor = new GroupByFunctionReplaceVisitor(functionMap); + groupBy.accept(replaceVisitor); + } + + for (SelectItem selectItem : plainSelect.getSelectItems()) { + selectItem.accept(visitor); + } + return selectStatement.toString(); + } public static String replaceFunction(String sql) { Select selectStatement = SqlParserSelectHelper.getSelect(sql); diff --git a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java index b4f9f3211..d15466bd3 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java @@ -34,5 +34,17 @@ class DateUtilsTest { dateStr = DateUtils.getBeforeDate("2023-08-10", 0, DatePeriodEnum.YEAR); Assert.assertEquals(dateStr, "2023-01-01"); + + dateStr = DateUtils.getBeforeDate(0, DatePeriodEnum.DAY); + //Assert.assertEquals(dateStr, "2023-09-08"); + + dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.DAY); + //Assert.assertEquals(dateStr, "2023-09-07"); + + dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.WEEK); + //Assert.assertEquals(dateStr, "2023-09-01"); + + dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.MONTH); + //Assert.assertEquals(dateStr, "2023-08-08"); } } \ No newline at end of file diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java index d26f9310b..df3bc6628 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java @@ -40,7 +40,6 @@ class SqlParserSelectHelperTest { + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); System.out.println(filterExpression); - filterExpression = SqlParserSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); @@ -70,6 +69,12 @@ class SqlParserSelectHelperTest { + "user_id = 'alice' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); System.out.println(filterExpression); + + filterExpression = SqlParserSelectHelper.getFilterExpression( + "SELECT department, user_id, field_a FROM s2 WHERE " + + "user_id like '%alice%' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); + + System.out.println(filterExpression); } @@ -93,6 +98,12 @@ class SqlParserSelectHelperTest { + " and 发布日期 ='11' group by 部门 limit 1"); Assert.assertEquals(allFields.size(), 5); + + allFields = SqlParserSelectHelper.getAllFields( + "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "); + + Assert.assertEquals(allFields.size(), 3); } @@ -135,6 +146,12 @@ class SqlParserSelectHelperTest { List selectFields = SqlParserSelectHelper.getOrderByFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); + + sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "; + selectFields = SqlParserSelectHelper.getOrderByFields(sql); + + Assert.assertEquals(selectFields.contains("pv"), true); } @@ -193,6 +210,11 @@ class SqlParserSelectHelperTest { hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, false); + + sql = "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " + + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; + hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql); + Assert.assertEquals(hasAggregateFunction, true); } private Map initParams() { diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelperTest.java index 33740931b..b9d46a879 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelperTest.java @@ -32,6 +32,18 @@ class SqlParserUpdateHelperTest { + "song_publis_date = '2023-08-01' AND publish_date >= '2023-08-08' " + "ORDER BY play_count DESC LIMIT 11", replaceSql); + replaceSql = "select MONTH(数据日期), sum(访问次数) from 内容库产品 " + + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " + + "group by MONTH(数据日期) order by sum(访问次数) desc limit 1"; + + replaceSql = SqlParserUpdateHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql); + + Assert.assertEquals( + "SELECT MONTH(sys_imp_date), sum(pv) FROM 内容库产品 WHERE sys_imp_date <= '2023-09-03' " + + "AND sys_imp_date >= '2023-03-03' " + + "GROUP BY MONTH(sys_imp_date) ORDER BY sum(pv) DESC LIMIT 1", replaceSql); + replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) " + "in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)"; @@ -180,6 +192,31 @@ class SqlParserUpdateHelperTest { } + + @Test + void replaceFunctionName() { + + String sql = "select MONTH(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; + Map functionMap = new HashMap<>(); + functionMap.put("MONTH".toLowerCase(), "toMonth"); + String replaceSql = SqlParserUpdateHelper.replaceFunction(sql, functionMap); + + Assert.assertEquals( + "SELECT toMonth(数据日期) AS 月份, avg(访问次数) AS 平均访问次数 FROM 内容库产品 WHERE" + + " datediff('month', 数据日期, '2023-09-02') <= 6 GROUP BY toMonth(数据日期)", + replaceSql); + + sql = "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; + replaceSql = SqlParserUpdateHelper.replaceFunction(sql, functionMap); + + Assert.assertEquals( + "SELECT toMonth(数据日期) AS 月份, avg(访问次数) AS 平均访问次数 FROM 内容库产品 WHERE" + + " datediff('month', 数据日期, '2023-09-02') <= 6 GROUP BY toMonth(数据日期)", + replaceSql); + } + private Map initParams() { Map fieldToBizName = new HashMap<>(); fieldToBizName.put("部门", "department"); @@ -192,7 +229,7 @@ class SqlParserUpdateHelperTest { fieldToBizName.put("播放", "play_count"); fieldToBizName.put("歌曲发布时间", "song_publis_date"); fieldToBizName.put("歌曲发布年份", "song_publis_year"); - fieldToBizName.put("转3.0前后30天结算份额衰减", "fdafdfdsa_fdas"); + fieldToBizName.put("访问次数", "pv"); return fieldToBizName; } } diff --git a/launchers/chat/src/main/resources/application-local.yaml b/launchers/chat/src/main/resources/application-local.yaml index 79f6da9dc..ce58b30a1 100644 --- a/launchers/chat/src/main/resources/application-local.yaml +++ b/launchers/chat/src/main/resources/application-local.yaml @@ -29,4 +29,4 @@ semantic: prefix: http://127.0.0.1:9081 llm: - url: http://127.0.0.1:9092 + url: http://127.0.0.1:9092/ diff --git a/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java b/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java index f33237c57..dd468a06a 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/advice/ResponseAdvice.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.advice; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.tencent.supersonic.common.pojo.ResultData; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -31,6 +32,7 @@ public class ResponseAdvice implements ResponseBodyAdvice { public Object beforeBodyWrite(Object result, MethodParameter methodParameter, MediaType mediaType, Class> aClass, ServerHttpRequest serverHttpRequest, ServerHttpResponse serverHttpResponse) { + objectMapper.registerModule(new JavaTimeModule()); if (result instanceof String) { return objectMapper.writeValueAsString(ResultData.success(result)); } diff --git a/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql b/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql index 3b25d76dd..e27fcd725 100644 --- a/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql +++ b/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql @@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `name` varchar(255) DEFAULT NULL , -- domain name `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID + `alias` varchar(255) DEFAULT NULL , -- internal name `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , `updated_at` TIMESTAMP DEFAULT NULL , diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java index a7cc93eeb..629b39a02 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java @@ -14,9 +14,11 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.plugin.Plugin; +import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.chat.service.QueryService; @@ -119,6 +121,16 @@ public class ConfigureDemo implements ApplicationListener chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); ItemVisibility visibility1 = new ItemVisibility(); chatAggConfig.setVisibility(visibility1); + List knowledgeInfos = new ArrayList<>(); + KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); + knowledgeInfoReq.setItemId(1L); + knowledgeInfoReq.setSearchEnable(true); + knowledgeInfos.add(knowledgeInfoReq); + KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); + knowledgeInfoReq2.setItemId(2L); + knowledgeInfoReq2.setSearchEnable(true); + knowledgeInfos.add(knowledgeInfoReq2); + chatAggConfig.setKnowledgeInfos(knowledgeInfos); chatConfigBaseReq.setChatAggConfig(chatAggConfig); List recommendedQuestions = new ArrayList<>(); @@ -170,13 +182,18 @@ public class ConfigureDemo implements ApplicationListener configService.addConfig(chatConfigBaseReq, user); } + private void addPlugin_1() { Plugin plugin1 = new Plugin(); plugin1.setType("WEB_PAGE"); plugin1.setModelList(Arrays.asList(1L)); plugin1.setPattern("用于分析超音数的流量概况，包含UV、PV等核心指标的追踪。P.S. 仅作为示例展示，无实际看板"); - plugin1.setParseModeConfig(null); plugin1.setName("超音数流量分析看板"); + PluginParseConfig pluginParseConfig = new PluginParseConfig(); + pluginParseConfig.setDescription(plugin1.getPattern()); + pluginParseConfig.setName(plugin1.getName()); + pluginParseConfig.setExamples(Lists.newArrayList("tom最近访问超音数情况怎么样")); + plugin1.setParseModeConfig(JSONObject.toJSONString(pluginParseConfig)); WebBase webBase = new WebBase(); webBase.setUrl("www.yourbi.com"); ParamOption paramOption = new ParamOption(); @@ -198,8 +215,8 @@ public class ConfigureDemo implements ApplicationListener agent.setDescription("帮助您用自然语言查询指标，支持时间限定、条件筛选、下钻维度以及聚合统计"); agent.setStatus(1); agent.setEnableSearch(1); - agent.setExamples(Lists.newArrayList("超音数访问次数", "近15天超音数访问次数汇总", - "按部门统计超音数的访问人数", "对比alice和lucy的停留时长", "超音数访问次数最高的部门")); + agent.setExamples(Lists.newArrayList("超音数访问次数", "近15天超音数访问次数汇总", "按部门统计超音数的访问人数", + "对比alice和lucy的停留时长", "超音数访问次数最高的部门")); AgentConfig agentConfig = new AgentConfig(); RuleQueryTool ruleQueryTool = new RuleQueryTool(); ruleQueryTool.setType(AgentToolType.RULE); diff --git a/launchers/standalone/src/main/resources/application-local.yaml b/launchers/standalone/src/main/resources/application-local.yaml index 8d831a929..bd552365d 100644 --- a/launchers/standalone/src/main/resources/application-local.yaml +++ b/launchers/standalone/src/main/resources/application-local.yaml @@ -38,3 +38,7 @@ mybatis: llm: url: http://127.0.0.1:9092 +embedding: + url: http://127.0.0.1:9092 +functionCall: + url: http://127.0.0.1:9092 diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 350b3cfb8..f433756a2 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -117,6 +117,7 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `name` varchar(255) DEFAULT NULL , -- domain name `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID + `alias` varchar(255) DEFAULT NULL , -- internal name `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , `updated_at` TIMESTAMP DEFAULT NULL , @@ -392,7 +393,21 @@ CREATE TABLE IF NOT EXISTS `singer` ( ); COMMENT ON TABLE singer IS 'singer_info'; - +CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( + `id` INT NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL , -- task name + `description` varchar(255) , + `command`LONGVARCHAR NOT NULL , -- task Request Parameters + `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 + `status` INT NOT NULL , -- the final status of the task + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `created_by` varchar(100) NOT NULL , + `progress` DOUBLE default 0.00 , -- task real-time progress + `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds + `message` LONGVARCHAR , -- remark related information + PRIMARY KEY (`id`) +); +COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index 34ea139b5..b56df3579 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -45,4 +45,8 @@ alter table s2_database add column viewer varchar(500); alter table s2_database drop column domain_id; --20230831 -alter table s2_chat add column agent_id int after chat_id; \ No newline at end of file +alter table s2_chat add column agent_id int after chat_id; + +--20230907 + +ALTER TABLE s2_model add alias varchar(200) default null after domain_id; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/optimization.properties b/launchers/standalone/src/main/resources/optimization.properties index bcdd883c4..bd4e14d4e 100644 --- a/launchers/standalone/src/main/resources/optimization.properties +++ b/launchers/standalone/src/main/resources/optimization.properties @@ -1,18 +1,10 @@ -#hanlp mapper helper - one.detection.size=8 one.detection.max.size=20 metric.dimension.min.threshold=0.3 metric.dimension.threshold=0.3 dimension.value.threshold=0.5 - function.bonus.threshold=201 - -#SatisfactionChecker long.text.threshold=0.8 short.text.threshold=0.5 query.text.length.threshold=10 - - candidate.threshold=0.2 - diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java index 8645896be..a5df68ec6 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; import com.tencent.supersonic.chat.plugin.PluginManager; -import com.tencent.supersonic.chat.query.metricinterpret.LLmAnswerResp; +import com.tencent.supersonic.chat.query.llm.interpret.LLmAnswerResp; import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.QueryService; import com.tencent.supersonic.util.DataUtils; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java index 2341b81ec..1993ed8d6 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java @@ -24,7 +24,7 @@ public class BasePluginTest { Assert.assertEquals(queryResult.getQueryMode(), WebPageQuery.QUERY_MODE); WebPageResponse webPageResponse = (WebPageResponse) queryResult.getResponse(); WebBaseResult webPage = webPageResponse.getWebPage(); - Assert.assertEquals(webPage.getUrl(), "www.test.com"); + Assert.assertEquals(webPage.getUrl(), "www.yourbi.com"); Assert.assertEquals(1, webPage.getParams().size()); Assert.assertEquals("alice", webPage.getParams().get(0).getValue()); } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java index 2bc35dec8..5142e48c2 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java @@ -36,7 +36,7 @@ public class PluginRecognizeTest extends BasePluginTest { public void webPageRecognize() throws Exception { MockConfiguration.mockEmbeddingRecognize(pluginManager, "alice最近的访问情况怎么样", "1"); MockConfiguration.mockEmbeddingUrl(embeddingConfig); - QueryReq queryContextReq = DataUtils.getQueryContextReq(1000, "alice最近的访问情况怎么样"); + QueryReq queryContextReq = DataUtils.getQueryReqWithAgent(1000, "alice最近的访问情况怎么样", 1); QueryResult queryResult = queryService.executeQuery(queryContextReq); assertPluginRecognizeResult(queryResult); } @@ -45,7 +45,7 @@ public class PluginRecognizeTest extends BasePluginTest { public void webPageRecognizeWithQueryFilter() throws Exception { MockConfiguration.mockEmbeddingRecognize(pluginManager, "在超音数最近的情况怎么样", "1"); MockConfiguration.mockEmbeddingUrl(embeddingConfig); - QueryReq queryRequest = DataUtils.getQueryContextReq(1000, "在超音数最近的情况怎么样"); + QueryReq queryRequest = DataUtils.getQueryReqWithAgent(1000, "在超音数最近的情况怎么样", 1); QueryFilters queryFilters = new QueryFilters(); QueryFilter queryFilter = new QueryFilter(); queryFilter.setElementID(2L); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 21923514f..0bb34a21c 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -91,7 +91,7 @@ public class DataUtils { dateInfo.setDateMode(dateMode); dateInfo.setPeriod(period); dateInfo.setStartDate(now().plusDays(-unit).toString()); - dateInfo.setEndDate(now().toString()); + dateInfo.setEndDate(now().plusDays(-1).toString()); return dateInfo; } diff --git a/launchers/standalone/src/test/resources/application-local.yaml b/launchers/standalone/src/test/resources/application-local.yaml index ef0671022..aecfef6b2 100644 --- a/launchers/standalone/src/test/resources/application-local.yaml +++ b/launchers/standalone/src/test/resources/application-local.yaml @@ -4,6 +4,8 @@ spring: path: /h2-console/chat # enabled web enabled: true + demo: + enabled: true datasource: driver-class-name: org.h2.Driver schema: classpath:db/schema-h2.sql diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 6cfe1a90c..64b84e38d 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -132,6 +132,7 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `name` varchar(255) DEFAULT NULL , -- domain name `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID + `alias` varchar(255) DEFAULT NULL , -- alias name `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , `updated_at` TIMESTAMP DEFAULT NULL , diff --git a/semantic/api/pom.xml b/semantic/api/pom.xml index 7c6b480e6..a501cf4bb 100644 --- a/semantic/api/pom.xml +++ b/semantic/api/pom.xml @@ -48,10 +48,6 @@ spring-core ${spring.version} - - com.google.guava - guava - com.alibaba fastjson @@ -78,4 +74,4 @@ ${project.version} - \ No newline at end of file + diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatabaseReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatabaseReq.java index 8533a21bd..fb2bdfc8c 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatabaseReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatabaseReq.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.semantic.api.model.request; import com.google.common.collect.Lists; import com.tencent.supersonic.semantic.api.model.enums.DataTypeEnum; import lombok.Data; -import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -37,7 +36,7 @@ public class DatabaseReq { private List viewers = Lists.newArrayList(); public String getUrl() { - if (StringUtils.isNotBlank(url)) { + if (type.equalsIgnoreCase(DataTypeEnum.H2.getFeature())) { return url; } if (type.equalsIgnoreCase(DataTypeEnum.MYSQL.getFeature())) { diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java index 21a96ad0b..c88bfefcf 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java @@ -1,13 +1,16 @@ package com.tencent.supersonic.semantic.api.model.request; +import com.google.common.collect.Lists; +import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import java.util.List; - +import java.util.stream.Collectors; import lombok.Data; +import org.springframework.util.CollectionUtils; @Data @@ -30,4 +33,14 @@ public class DatasourceReq extends SchemaItem { private List measures; + + public List getTimeDimension() { + if (CollectionUtils.isEmpty(dimensions)) { + return Lists.newArrayList(); + } + return dimensions.stream() + .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType())) + .collect(Collectors.toList()); + } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java index a900b00b1..20413eb79 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java @@ -16,6 +16,8 @@ public class ModelReq extends SchemaItem { private Integer isOpen = 0; + private String alias; + private List viewers = new ArrayList<>(); private List viewOrgs = new ArrayList<>(); diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java index 7c24171c1..b2580779d 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java @@ -8,7 +8,7 @@ import java.util.List; @Data public class PageSchemaItemReq extends PageBaseReq { - private Long id; + private String id; private String name; private String bizName; private String createdBy; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java index f3751d794..ff6e484df 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java @@ -14,4 +14,5 @@ public class DatasourceResp extends SchemaItem { private DatasourceDetail datasourceDetail; + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainResp.java index 13c66160c..809f0d061 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainResp.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.Entity; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import java.util.List; +import java.util.Objects; import lombok.Data; import lombok.ToString; @@ -32,4 +33,25 @@ public class DomainResp extends SchemaItem { private Entity entity; + private boolean hasEditPermission = false; + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + DomainResp that = (DomainResp) o; + if (getId() == null || that.getId() == null) { + return false; + } + return Objects.equals(getId().intValue(), that.getId().intValue()); + } + + @Override + public int hashCode() { + if (getId() == null) { + return 0; + } + return getId().hashCode(); + } } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java index 1e1e9b8c4..a6337881c 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java @@ -13,6 +13,8 @@ public class ModelResp extends SchemaItem { private Long domainId; + private String alias; + private List viewers; private List viewOrgs; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java index ee02f712a..61a922820 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java @@ -22,4 +22,5 @@ public class DatasourceYamlTpl { private List measures; + } diff --git a/semantic/model/pom.xml b/semantic/model/pom.xml index 264bfe3ae..254ed4aa4 100644 --- a/semantic/model/pom.xml +++ b/semantic/model/pom.xml @@ -34,6 +34,11 @@ ${project.version} compile + + com.tencent.supersonic + common + ${project.version} + com.fasterxml.jackson.datatype jackson-datatype-jsr310 @@ -73,9 +78,6 @@ ${pagehelper.version} compile - - - org.testng testng @@ -98,4 +100,4 @@ - \ No newline at end of file + diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java index d68c256ad..067b8af44 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java @@ -21,7 +21,6 @@ import java.util.Map; import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; @Slf4j @Component @@ -49,12 +48,7 @@ public class CatalogImpl implements Catalog { } public DatabaseResp getDatabaseByModelId(Long modelId) { - List datasourceResps = datasourceService.getDatasourceList(modelId); - if (!CollectionUtils.isEmpty(datasourceResps)) { - Long databaseId = datasourceResps.iterator().next().getDatabaseId(); - return databaseService.getDatabase(databaseId); - } - return null; + return modelService.getDatabaseByModelId(modelId); } @Override @@ -77,7 +71,7 @@ public class CatalogImpl implements Catalog { } @Override - public void getModelYamlTplByMoldelIds(Set modelIds, Map> dimensionYamlMap, + public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, List datasourceYamlTplList, List metricYamlTplList) { datasourceService.getModelYamlTplByModelIds(modelIds, dimensionYamlMap, datasourceYamlTplList, metricYamlTplList); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java index 598d40290..6c4e731a3 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java @@ -77,7 +77,7 @@ public class DatabaseServiceImpl implements DatabaseService { databaseResp.setHasEditPermission(true); databaseResp.setHasUsePermission(true); } - if (databaseResp.getViewers().contains(databaseResp.getCreatedBy())) { + if (databaseResp.getViewers().contains(user.getName())) { databaseResp.setHasUsePermission(true); } }); @@ -108,8 +108,10 @@ public class DatabaseServiceImpl implements DatabaseService { return new QueryResultWithSchemaResp(); } List admins = databaseResp.getAdmins(); + List viewers = databaseResp.getViewers(); if (!admins.contains(user.getName()) - || !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())) { + && !viewers.contains(user.getName()) + && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())) { String message = String.format("您暂无当前数据库%s权限, 请联系数据库管理员%s开通", databaseResp.getName(), String.join(",", admins)); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java index 4073f83cf..a1b20821b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.Dim; +import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; @@ -175,8 +176,17 @@ public class DatasourceServiceImpl implements DatasourceService { private void preCheck(DatasourceReq datasourceReq) { List dims = datasourceReq.getDimensions(); + List measures = datasourceReq.getMeasures(); + List timeDims = datasourceReq.getTimeDimension(); + List identifies = datasourceReq.getIdentifiers(); if (CollectionUtils.isEmpty(dims)) { - throw new RuntimeException("lack of dimension"); + throw new RuntimeException("缺少维度信息"); + } + if (CollectionUtils.isEmpty(identifies)) { + throw new RuntimeException("缺少主键信息"); + } + if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) { + throw new RuntimeException("有度量时, 不可缺少时间维度"); } } @@ -241,7 +251,7 @@ public class DatasourceServiceImpl implements DatasourceService { List metricResps = metricService.getMetrics(modelId, datasourceId); List dimensionResps = dimensionService.getDimensionsByDatasource(datasourceId); if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(dimensionResps)) { - throw new RuntimeException("exist dimension or metric on this datasource, please check"); + throw new RuntimeException("存在基于该数据源创建的指标和维度, 暂不能删除, 请确认"); } } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index 9bc49594f..4d372677e 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -7,6 +7,10 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DataAddEvent; +import com.tencent.supersonic.common.pojo.DataDeleteEvent; +import com.tencent.supersonic.common.pojo.DataUpdateEvent; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; @@ -18,12 +22,12 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.model.domain.DatabaseService; +import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO; import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository; import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter; import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.pojo.Dimension; import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; @@ -35,6 +39,8 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -48,19 +54,22 @@ public class DimensionServiceImpl implements DimensionService { private DatasourceService datasourceService; - private DomainService domainService; + private ModelService modelService; private ChatGptHelper chatGptHelper; private DatabaseService databaseService; + @Autowired + private ApplicationEventPublisher applicationEventPublisher; + public DimensionServiceImpl(DimensionRepository dimensionRepository, - DomainService domainService, + ModelService modelService, DatasourceService datasourceService, ChatGptHelper chatGptHelper, DatabaseService databaseService) { - this.domainService = domainService; + this.modelService = modelService; this.dimensionRepository = dimensionRepository; this.datasourceService = datasourceService; this.chatGptHelper = chatGptHelper; @@ -75,6 +84,11 @@ public class DimensionServiceImpl implements DimensionService { log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension)); dimension.createdBy(user.getName()); saveDimension(dimension); + //动态更新字典 + String type = DictWordType.DIMENSION.getType(); + DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId()); + applicationEventPublisher.publishEvent( + new DataAddEvent(this, dimension.getName(), dimension.getModelId(), dimensionResp.getId(), type)); } @@ -102,7 +116,20 @@ public class DimensionServiceImpl implements DimensionService { Dimension dimension = DimensionConverter.convert(dimensionReq); dimension.updatedBy(user.getName()); log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension)); + List dimensionRespList = getDimensions(dimensionReq.getModelId()).stream().filter( + o -> o.getId().equals(dimensionReq.getId())).collect(Collectors.toList()); updateDimension(dimension); + //动态更新字典 + String type = DictWordType.DIMENSION.getType(); + if (!CollectionUtils.isEmpty(dimensionRespList)) { + log.info("dimensionRespList size:{}", dimensionRespList.size()); + log.info("name:{}", dimensionRespList.get(0).getName()); + applicationEventPublisher.publishEvent( + new DataUpdateEvent(this, dimensionRespList.get(0).getName(), + dimensionReq.getName(), + dimension.getModelId(), + dimensionRespList.get(0).getId(), type)); + } } protected void updateDimension(Dimension dimension) { @@ -148,10 +175,10 @@ public class DimensionServiceImpl implements DimensionService { public List getDimensions(List ids) { List dimensionResps = Lists.newArrayList(); List dimensionDOS = dimensionRepository.getDimensionListByIds(ids); - Map fullDomainPathMap = domainService.getDomainFullPath(); + Map modelFullPathMap = modelService.getModelFullPathMap(); if (!CollectionUtils.isEmpty(dimensionDOS)) { dimensionResps = dimensionDOS.stream() - .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, fullDomainPathMap, + .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelFullPathMap, new HashMap<>())) .collect(Collectors.toList()); } @@ -184,10 +211,10 @@ public class DimensionServiceImpl implements DimensionService { private List convertList(List dimensionDOS, Map datasourceRespMap) { List dimensionResps = Lists.newArrayList(); - Map fullDomainPathMap = domainService.getDomainFullPath(); + Map modelFullPathMap = modelService.getModelFullPathMap(); if (!CollectionUtils.isEmpty(dimensionDOS)) { dimensionResps = dimensionDOS.stream() - .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, fullDomainPathMap, + .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelFullPathMap, datasourceRespMap)) .collect(Collectors.toList()); } @@ -256,6 +283,11 @@ public class DimensionServiceImpl implements DimensionService { throw new RuntimeException(String.format("the dimension %s not exist", id)); } dimensionRepository.deleteDimension(id); + //动态更新字典 + String type = DictWordType.DIMENSION.getType(); + applicationEventPublisher.publishEvent( + new DataDeleteEvent(this, dimensionDO.getName(), dimensionDO.getModelId(), dimensionDO.getId(), type)); + } @Override @@ -300,10 +332,19 @@ public class DimensionServiceImpl implements DimensionService { int i = 0; for (Map stringObjectMap : resultList) { DimValueMap dimValueMap = new DimValueMap(); - dimValueMap.setTechName((String) stringObjectMap.get(dimensionReq.getBizName())); - dimValueMap.setBizName(jsonObject.getJSONArray("tran").getString(i)); - dimValueMap.setAlias(jsonObject.getJSONObject("alias").getJSONArray( - (String) stringObjectMap.get(dimensionReq.getBizName())).toJavaList(String.class)); + dimValueMap.setTechName(String.valueOf(stringObjectMap.get(dimensionReq.getBizName()))); + try { + String tran = jsonObject.getJSONArray("tran").getString(i); + dimValueMap.setBizName(tran); + } catch (Exception exception) { + dimValueMap.setBizName(""); + } + try { + dimValueMap.setAlias(jsonObject.getJSONObject("alias") + .getJSONArray(stringObjectMap.get(dimensionReq.getBizName()) + "").toJavaList(String.class)); + } catch (Exception exception) { + dimValueMap.setAlias(null); + } dimValueMapsResp.add(dimValueMap); i++; } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java index 162f3b0b1..fd51d9496 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java @@ -75,6 +75,10 @@ public class DomainServiceImpl implements DomainService { @Override public void deleteDomain(Long id) { + List modelResps = modelService.getModelByDomainIds(Lists.newArrayList(id)); + if (!CollectionUtils.isEmpty(modelResps)) { + throw new RuntimeException("该主题域下还存在模型, 暂不能删除, 请确认"); + } domainRepository.deleteDomain(id); } @@ -124,7 +128,9 @@ public class DomainServiceImpl implements DomainService { List domainIds = domainWithAuth.stream().map(DomainResp::getId) .collect(Collectors.toList()); //get all child domain - return getDomainChildren(domainIds); + return getDomainChildren(domainIds).stream() + .peek(domainResp -> domainResp.setHasEditPermission(true)) + .collect(Collectors.toSet()); } private Set getParentDomain(List ids) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index 3cb2e9982..6545e1b70 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -6,6 +6,10 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DataAddEvent; +import com.tencent.supersonic.common.pojo.DataDeleteEvent; +import com.tencent.supersonic.common.pojo.DataUpdateEvent; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; @@ -23,14 +27,18 @@ import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository; import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter; import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.model.domain.pojo.Metric; + import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -46,6 +54,9 @@ public class MetricServiceImpl implements MetricService { private ChatGptHelper chatGptHelper; + @Autowired + private ApplicationEventPublisher applicationEventPublisher; + public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, DomainService domainService, @@ -63,6 +74,11 @@ public class MetricServiceImpl implements MetricService { metric.createdBy(user.getName()); log.info("[create metric] object:{}", JSONObject.toJSONString(metric)); saveMetric(metric); + //动态更新字典 + String type = DictWordType.METRIC.getType(); + MetricResp metricResp = getMetric(metric.getModelId(), metric.getBizName()); + applicationEventPublisher.publishEvent( + new DataAddEvent(this, metric.getName(), metric.getModelId(), metricResp.getId(), type)); } @Override @@ -150,7 +166,21 @@ public class MetricServiceImpl implements MetricService { Metric metric = MetricConverter.convert(metricReq); metric.updatedBy(user.getName()); log.info("[update metric] object:{}", JSONObject.toJSONString(metric)); + List metricRespList = getMetrics(metricReq.getModelId()).stream().filter( + o -> o.getId().equals(metricReq.getId())).collect(Collectors.toList()); updateMetric(metric); + //动态更新字典 + String type = DictWordType.METRIC.getType(); + //MetricResp metricResp = getMetric(metric.getModelId(), metric.getBizName()); + if (!CollectionUtils.isEmpty(metricRespList)) { + log.info("metricRespList size:{}", metricRespList.size()); + log.info("name:{}", metricRespList.get(0).getName()); + applicationEventPublisher.publishEvent( + new DataUpdateEvent(this, metricRespList.get(0).getName(), + metricReq.getName(), + metric.getModelId(), + metricRespList.get(0).getId(), type)); + } } public void saveMetric(Metric metric) { @@ -205,6 +235,10 @@ public class MetricServiceImpl implements MetricService { throw new RuntimeException(String.format("the metric %s not exist", id)); } metricRepository.deleteMetric(id); + //动态更新字典 + String type = DictWordType.METRIC.getType(); + applicationEventPublisher.publishEvent( + new DataDeleteEvent(this, metricDO.getName(), metricDO.getModelId(), metricDO.getId(), type)); } @Override @@ -217,7 +251,6 @@ public class MetricServiceImpl implements MetricService { } - private void saveMetricBatch(List metrics, User user) { if (CollectionUtils.isEmpty(metrics)) { return; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 54b3c2b97..b10dd9704 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -9,6 +9,7 @@ import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; +import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; @@ -17,6 +18,7 @@ import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; +import com.tencent.supersonic.semantic.model.domain.DatabaseService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.DimensionService; @@ -51,16 +53,19 @@ public class ModelServiceImpl implements ModelService { private final DatasourceService datasourceService; private final DomainService domainService; private final UserService userService; + private final DatabaseService databaseService; public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService, @Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService, - @Lazy DomainService domainService, UserService userService) { + @Lazy DomainService domainService, UserService userService, + @Lazy DatabaseService databaseService) { this.modelRepository = modelRepository; this.metricService = metricService; this.dimensionService = dimensionService; this.datasourceService = datasourceService; this.domainService = domainService; this.userService = userService; + this.databaseService = databaseService; } @Override @@ -172,7 +177,7 @@ public class ModelServiceImpl implements ModelService { List datasourceResps = datasourceService.getDatasourceList(id); if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(datasourceResps) || !CollectionUtils.isEmpty(dimensionResps)) { - throw new RuntimeException("exist datasource, dimension or metric in this model, please check"); + throw new RuntimeException("该模型下存在数据源、指标或者维度, 暂不能删除, 请确认"); } } @@ -277,6 +282,16 @@ public class ModelServiceImpl implements ModelService { return modelSchemaRespList; } + @Override + public DatabaseResp getDatabaseByModelId(Long modelId) { + List datasourceResps = datasourceService.getDatasourceList(modelId); + if (!CollectionUtils.isEmpty(datasourceResps)) { + Long databaseId = datasourceResps.iterator().next().getDatabaseId(); + return databaseService.getDatabase(databaseId); + } + return null; + } + private List generateMetricSchema(Long modelId) { List metricSchemaDescList = new ArrayList<>(); List metricDescList = metricService.getMetrics(modelId); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java index 90d30755f..b1c7fca3d 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java @@ -30,7 +30,7 @@ public interface Catalog { List getMetrics(Long modelId); - void getModelYamlTplByMoldelIds(Set modelIds, Map> dimensionYamlMap, + void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, List datasourceYamlTplList, List metricYamlTplList); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java index a100689dd..7e05fa38e 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; +import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; @@ -39,4 +40,6 @@ public interface ModelService { ModelSchemaResp fetchSingleModelSchema(Long modelId); List fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq); + + DatabaseResp getDatabaseByModelId(Long modelId); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/ClickHouseAdaptor.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/ClickHouseAdaptor.java index 4a3d48e7a..f4e8b4b9a 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/ClickHouseAdaptor.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/ClickHouseAdaptor.java @@ -1,7 +1,10 @@ package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.Constants; +import java.util.HashMap; +import java.util.Map; public class ClickHouseAdaptor extends EngineAdaptor { @@ -40,6 +43,15 @@ public class ClickHouseAdaptor extends EngineAdaptor { return "select name from system.tables where database = '%s';"; } + @Override + public String functionNameCorrector(String sql) { + Map functionMap = new HashMap<>(); + functionMap.put("MONTH".toLowerCase(), "toMonth"); + functionMap.put("DAY".toLowerCase(), "toDayOfMonth"); + functionMap.put("YEAR".toLowerCase(), "toYear"); + return SqlParserUpdateHelper.replaceFunction(sql, functionMap); + } + @Override public String getColumnMetaQueryTpl() { return "select name,type as dataType, comment from system.columns where database = '%s' and table='%s'"; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/EngineAdaptor.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/EngineAdaptor.java index 63af21df2..d9d46b3a8 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/EngineAdaptor.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/EngineAdaptor.java @@ -12,4 +12,6 @@ public abstract class EngineAdaptor { public abstract String getDbMetaQueryTpl(); public abstract String getTableMetaQueryTpl(); + + public abstract String functionNameCorrector(String sql); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/H2Adaptor.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/H2Adaptor.java index 20dce0613..b2295d397 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/H2Adaptor.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/H2Adaptor.java @@ -43,4 +43,9 @@ public class H2Adaptor extends EngineAdaptor { return "SELECT TABLE_NAME as name FROM INFORMATION_SCHEMA.TABLES " + "WHERE STORAGE_TYPE = 'MEMORY' AND TABLE_SCHEMA = '%s'"; } + + @Override + public String functionNameCorrector(String sql) { + return sql; + } } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/MysqlAdaptor.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/MysqlAdaptor.java index 03475dcd7..b225c477b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/MysqlAdaptor.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/MysqlAdaptor.java @@ -44,6 +44,11 @@ public class MysqlAdaptor extends EngineAdaptor { return "select TABLE_NAME as name from information_schema.tables where TABLE_SCHEMA = '%s';"; } + @Override + public String functionNameCorrector(String sql) { + return sql; + } + @Override public String getColumnMetaQueryTpl() { return "SELECT COLUMN_NAME as name, DATA_TYPE as dataType, COLUMN_COMMENT as comment " diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/config/YamlConfig.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/config/YamlConfig.java deleted file mode 100644 index 218a95089..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/config/YamlConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.config; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - - -@Configuration -public class YamlConfig { - - - @Value("${model.yaml.file.dir: conf/models/}") - private String metaYamlFileDir; - - public String getmetaYamlFileDir() { - return metaYamlFileDir; - } - -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java index 7aef7702b..883d24070 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java @@ -58,6 +58,7 @@ public class DatasourceDO { */ private String datasourceDetail; + /** * @return id */ @@ -251,4 +252,6 @@ public class DatasourceDO { public void setDatasourceDetail(String datasourceDetail) { this.datasourceDetail = datasourceDetail == null ? null : datasourceDetail.trim(); } + + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java index baa07e3db..b05e1a7a9 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java @@ -3,144 +3,318 @@ package com.tencent.supersonic.semantic.model.domain.dataobject; import java.util.Date; public class ModelDO { - + /** + * + */ private Long id; + /** + * + */ private String name; + /** + * + */ private String bizName; + /** + * + */ private Long domainId; + /** + * + */ + private String alias; + + /** + * + */ private String viewer; + /** + * + */ private String viewOrg; + /** + * + */ private String admin; + /** + * + */ private String adminOrg; + /** + * + */ private Integer isOpen; + /** + * + */ private String createdBy; + /** + * + */ private Date createdAt; + /** + * + */ private String updatedBy; + /** + * + */ private Date updatedAt; + /** + * + */ private String entity; + /** + * + * @return id + */ public Long getId() { return id; } + /** + * + * @param id + */ public void setId(Long id) { this.id = id; } + /** + * + * @return name + */ public String getName() { return name; } + /** + * + * @param name + */ public void setName(String name) { this.name = name == null ? null : name.trim(); } + /** + * + * @return biz_name + */ public String getBizName() { return bizName; } + /** + * + * @param bizName + */ public void setBizName(String bizName) { this.bizName = bizName == null ? null : bizName.trim(); } + /** + * + * @return domain_id + */ public Long getDomainId() { return domainId; } + /** + * + * @param domainId + */ public void setDomainId(Long domainId) { this.domainId = domainId; } + /** + * + * @return alias + */ + public String getAlias() { + return alias; + } + + /** + * + * @param alias + */ + public void setAlias(String alias) { + this.alias = alias == null ? null : alias.trim(); + } + + /** + * + * @return viewer + */ public String getViewer() { return viewer; } + /** + * + * @param viewer + */ public void setViewer(String viewer) { this.viewer = viewer == null ? null : viewer.trim(); } + /** + * + * @return view_org + */ public String getViewOrg() { return viewOrg; } + /** + * + * @param viewOrg + */ public void setViewOrg(String viewOrg) { this.viewOrg = viewOrg == null ? null : viewOrg.trim(); } + /** + * + * @return admin + */ public String getAdmin() { return admin; } + /** + * + * @param admin + */ public void setAdmin(String admin) { this.admin = admin == null ? null : admin.trim(); } + /** + * + * @return admin_org + */ public String getAdminOrg() { return adminOrg; } + /** + * + * @param adminOrg + */ public void setAdminOrg(String adminOrg) { this.adminOrg = adminOrg == null ? null : adminOrg.trim(); } + /** + * + * @return is_open + */ public Integer getIsOpen() { return isOpen; } + /** + * + * @param isOpen + */ public void setIsOpen(Integer isOpen) { this.isOpen = isOpen; } + /** + * + * @return created_by + */ public String getCreatedBy() { return createdBy; } + /** + * + * @param createdBy + */ public void setCreatedBy(String createdBy) { this.createdBy = createdBy == null ? null : createdBy.trim(); } + /** + * + * @return created_at + */ public Date getCreatedAt() { return createdAt; } + /** + * + * @param createdAt + */ public void setCreatedAt(Date createdAt) { this.createdAt = createdAt; } + /** + * + * @return updated_by + */ public String getUpdatedBy() { return updatedBy; } + /** + * + * @param updatedBy + */ public void setUpdatedBy(String updatedBy) { this.updatedBy = updatedBy == null ? null : updatedBy.trim(); } + /** + * + * @return updated_at + */ public Date getUpdatedAt() { return updatedAt; } + /** + * + * @param updatedAt + */ public void setUpdatedAt(Date updatedAt) { this.updatedAt = updatedAt; } + /** + * + * @return entity + */ public String getEntity() { return entity; } + /** + * + * @param entity + */ public void setEntity(String entity) { this.entity = entity == null ? null : entity.trim(); } -} +} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java index 793478173..9744d7b39 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java @@ -31,6 +31,7 @@ public class ModelDOExample { protected Integer limitEnd; /** + * * @mbg.generated */ public ModelDOExample() { @@ -38,6 +39,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public void setOrderByClause(String orderByClause) { @@ -45,6 +47,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public String getOrderByClause() { @@ -52,6 +55,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public void setDistinct(boolean distinct) { @@ -59,6 +63,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public boolean isDistinct() { @@ -66,6 +71,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public List getOredCriteria() { @@ -73,6 +79,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public void or(Criteria criteria) { @@ -80,6 +87,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public Criteria or() { @@ -89,6 +97,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public Criteria createCriteria() { @@ -100,6 +109,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ protected Criteria createCriteriaInternal() { @@ -108,6 +118,7 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public void clear() { @@ -117,13 +128,15 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public void setLimitStart(Integer limitStart) { - this.limitStart = limitStart; + this.limitStart=limitStart; } /** + * * @mbg.generated */ public Integer getLimitStart() { @@ -131,13 +144,15 @@ public class ModelDOExample { } /** + * * @mbg.generated */ public void setLimitEnd(Integer limitEnd) { - this.limitEnd = limitEnd; + this.limitEnd=limitEnd; } /** + * * @mbg.generated */ public Integer getLimitEnd() { @@ -448,6 +463,76 @@ public class ModelDOExample { return (Criteria) this; } + public Criteria andAliasIsNull() { + addCriterion("alias is null"); + return (Criteria) this; + } + + public Criteria andAliasIsNotNull() { + addCriterion("alias is not null"); + return (Criteria) this; + } + + public Criteria andAliasEqualTo(String value) { + addCriterion("alias =", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasNotEqualTo(String value) { + addCriterion("alias <>", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasGreaterThan(String value) { + addCriterion("alias >", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasGreaterThanOrEqualTo(String value) { + addCriterion("alias >=", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasLessThan(String value) { + addCriterion("alias <", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasLessThanOrEqualTo(String value) { + addCriterion("alias <=", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasLike(String value) { + addCriterion("alias like", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasNotLike(String value) { + addCriterion("alias not like", value, "alias"); + return (Criteria) this; + } + + public Criteria andAliasIn(List values) { + addCriterion("alias in", values, "alias"); + return (Criteria) this; + } + + public Criteria andAliasNotIn(List values) { + addCriterion("alias not in", values, "alias"); + return (Criteria) this; + } + + public Criteria andAliasBetween(String value1, String value2) { + addCriterion("alias between", value1, value2, "alias"); + return (Criteria) this; + } + + public Criteria andAliasNotBetween(String value1, String value2) { + addCriterion("alias not between", value1, value2, "alias"); + return (Criteria) this; + } + public Criteria andViewerIsNull() { addCriterion("viewer is null"); return (Criteria) this; @@ -1079,6 +1164,38 @@ public class ModelDOExample { private String typeHandler; + public String getCondition() { + return condition; + } + + public Object getValue() { + return value; + } + + public Object getSecondValue() { + return secondValue; + } + + public boolean isNoValue() { + return noValue; + } + + public boolean isSingleValue() { + return singleValue; + } + + public boolean isBetweenValue() { + return betweenValue; + } + + public boolean isListValue() { + return listValue; + } + + public String getTypeHandler() { + return typeHandler; + } + protected Criterion(String condition) { super(); this.condition = condition; @@ -1114,37 +1231,5 @@ public class ModelDOExample { protected Criterion(String condition, Object value, Object secondValue) { this(condition, value, secondValue, null); } - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } } -} +} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java index 74f78b4b1..a9ba48b4c 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java @@ -4,17 +4,17 @@ import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.Measure; +import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl; -import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.model.domain.utils.SysTimeDimensionBuilder; import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptor; import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptorFactory; import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; import com.tencent.supersonic.semantic.model.domain.pojo.DatasourceQueryEnum; +import com.tencent.supersonic.semantic.model.domain.utils.SysTimeDimensionBuilder; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java index 286512309..6737381b6 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java @@ -16,4 +16,5 @@ public class Datasource extends SchemaItem { private DatasourceDetail datasourceDetail; + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java index 947fb0ae0..02d93bd88 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java @@ -7,7 +7,7 @@ import java.util.List; @Data public class MetaFilter { - private Long id; + private String id; private String name; @@ -21,4 +21,6 @@ public class MetaFilter { private Integer status; + private String key; + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetricFilter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetricFilter.java index 38ac698e3..334f2f453 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetricFilter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetricFilter.java @@ -7,6 +7,4 @@ public class MetricFilter extends MetaFilter { private String type; - private String key; - } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Model.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Model.java index 299384ddf..971919bc2 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Model.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Model.java @@ -18,6 +18,8 @@ public class Model extends SchemaItem { private Integer isOpen; + private String alias; + private List viewers; private List viewOrgs; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java index 67e8f7e9a..4b07cde55 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java @@ -52,7 +52,7 @@ public class DimensionConverter { Map datasourceRespMap) { DimensionResp dimensionResp = new DimensionResp(); BeanUtils.copyProperties(dimensionDO, dimensionResp); - dimensionResp.setFullPath(fullPathMap.get(dimensionDO.getModelId()) + dimensionDO.getBizName()); + dimensionResp.setFullPath(fullPathMap.get(dimensionDO.getModelId()) + "/" + dimensionDO.getBizName()); dimensionResp.setDatasourceId( datasourceRespMap.getOrDefault(dimensionResp.getDatasourceId(), new DatasourceResp()).getId()); dimensionResp.setDatasourceName( diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/SqlUtils.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/SqlUtils.java index ea2f71963..be7782c04 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/SqlUtils.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/SqlUtils.java @@ -2,8 +2,9 @@ package com.tencent.supersonic.semantic.model.domain.utils; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; -import com.tencent.supersonic.semantic.api.model.enums.DataTypeEnum; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.semantic.api.model.enums.DataTypeEnum; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.model.domain.pojo.JdbcDataSource; @@ -12,7 +13,11 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -147,11 +152,26 @@ public class SqlUtils { for (QueryColumn queryColumn : queryColumns) { String colName = queryColumn.getNameEn(); Object value = rs.getObject(colName); - map.put(colName, value instanceof byte[] ? new String((byte[]) value) : value); + map.put(colName, getValue(value)); } return map; } + private Object getValue(Object value) { + if (value instanceof LocalDate) { + LocalDate localDate = (LocalDate) value; + return localDate.format(DateTimeFormatter.ofPattern(DateUtils.DATE_FORMAT)); + } else if (value instanceof LocalDateTime) { + LocalDateTime localDateTime = (LocalDateTime) value; + return localDateTime.format(DateTimeFormatter.ofPattern(DateUtils.TIME_FORMAT)); + } else if (value instanceof Date) { + Date date = (Date) value; + return DateUtils.format(date); + } else if (value instanceof byte[]) { + return new String((byte[]) value); + } + return value; + } public static final class SqlUtilsBuilder { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java index f9bb9a746..a2bcebea3 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.semantic.model.infrastructure.mapper; import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO; - import java.util.List; +import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import org.apache.ibatis.annotations.Mapper; @Mapper @@ -13,4 +13,5 @@ public interface DimensionDOCustomMapper { void batchUpdate(List dimensionDOS); + List query(DimensionFilter dimensionFilter); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java index 8102d28d7..16ad69922 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java @@ -6,10 +6,7 @@ import com.tencent.supersonic.semantic.model.domain.repository.DimensionReposito import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOCustomMapper; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOMapper; - import java.util.List; - -import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; @@ -84,27 +81,7 @@ public class DimensionRepositoryImpl implements DimensionRepository { @Override public List getDimension(DimensionFilter dimensionFilter) { - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - dimensionDOExample.createCriteria(); - if (dimensionFilter.getId() != null) { - dimensionDOExample.getOredCriteria().get(0).andIdEqualTo(dimensionFilter.getId()); - } - if (dimensionFilter.getName() != null) { - dimensionDOExample.getOredCriteria().get(0).andNameLike("%" + dimensionFilter.getName() + "%"); - } - if (dimensionFilter.getBizName() != null) { - dimensionDOExample.getOredCriteria().get(0).andBizNameLike("%" + dimensionFilter.getBizName() + "%"); - } - if (dimensionFilter.getCreatedBy() != null) { - dimensionDOExample.getOredCriteria().get(0).andCreatedByEqualTo(dimensionFilter.getCreatedBy()); - } - if (CollectionUtils.isNotEmpty(dimensionFilter.getModelIds())) { - dimensionDOExample.getOredCriteria().get(0).andModelIdIn(dimensionFilter.getModelIds()); - } - if (dimensionFilter.getSensitiveLevel() != null) { - dimensionDOExample.getOredCriteria().get(0).andSensitiveLevelEqualTo(dimensionFilter.getSensitiveLevel()); - } - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); + return dimensionDOCustomMapper.query(dimensionFilter); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java index 70089dd80..0eb0b5175 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ModelReq; +import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.model.domain.ModelService; import org.springframework.web.bind.annotation.RestController; @@ -74,4 +75,9 @@ public class ModelController { .collect(Collectors.toList())); } + @GetMapping("/getModelDatabase/{modelId}") + public DatabaseResp getModelDatabase(@PathVariable("modelId") Long modelId) { + return modelService.getDatabaseByModelId(modelId); + } + } diff --git a/semantic/model/src/main/resources/mapper/ModelDOMapper.xml b/semantic/model/src/main/resources/mapper/ModelDOMapper.xml index 4b43dd104..85ae94f2d 100644 --- a/semantic/model/src/main/resources/mapper/ModelDOMapper.xml +++ b/semantic/model/src/main/resources/mapper/ModelDOMapper.xml @@ -6,6 +6,7 @@ + @@ -78,8 +79,8 @@ - id, name, biz_name, domain_id, viewer, view_org, admin, admin_org, is_open, created_by, - created_at, updated_by, updated_at + id, name, biz_name, domain_id, alias, viewer, view_org, admin, admin_org, is_open, + created_by, created_at, updated_by, updated_at entity @@ -131,15 +132,17 @@ insert into s2_model (id, name, biz_name, - domain_id, viewer, view_org, - admin, admin_org, is_open, - created_by, created_at, updated_by, - updated_at, entity) + domain_id, alias, viewer, + view_org, admin, admin_org, + is_open, created_by, created_at, + updated_by, updated_at, entity + ) values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{bizName,jdbcType=VARCHAR}, - #{domainId,jdbcType=BIGINT}, #{viewer,jdbcType=VARCHAR}, #{viewOrg,jdbcType=VARCHAR}, - #{admin,jdbcType=VARCHAR}, #{adminOrg,jdbcType=VARCHAR}, #{isOpen,jdbcType=INTEGER}, - #{createdBy,jdbcType=VARCHAR}, #{createdAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR}, - #{updatedAt,jdbcType=TIMESTAMP}, #{entity,jdbcType=LONGVARCHAR}) + #{domainId,jdbcType=BIGINT}, #{alias,jdbcType=VARCHAR}, #{viewer,jdbcType=VARCHAR}, + #{viewOrg,jdbcType=VARCHAR}, #{admin,jdbcType=VARCHAR}, #{adminOrg,jdbcType=VARCHAR}, + #{isOpen,jdbcType=INTEGER}, #{createdBy,jdbcType=VARCHAR}, #{createdAt,jdbcType=TIMESTAMP}, + #{updatedBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, #{entity,jdbcType=LONGVARCHAR} + ) insert into s2_model @@ -156,6 +159,9 @@ domain_id, + + alias, + viewer, @@ -200,6 +206,9 @@ #{domainId,jdbcType=BIGINT}, + + #{alias,jdbcType=VARCHAR}, + #{viewer,jdbcType=VARCHAR}, @@ -253,6 +262,9 @@ domain_id = #{record.domainId,jdbcType=BIGINT}, + + alias = #{record.alias,jdbcType=VARCHAR}, + viewer = #{record.viewer,jdbcType=VARCHAR}, @@ -294,6 +306,7 @@ name = #{record.name,jdbcType=VARCHAR}, biz_name = #{record.bizName,jdbcType=VARCHAR}, domain_id = #{record.domainId,jdbcType=BIGINT}, + alias = #{record.alias,jdbcType=VARCHAR}, viewer = #{record.viewer,jdbcType=VARCHAR}, view_org = #{record.viewOrg,jdbcType=VARCHAR}, admin = #{record.admin,jdbcType=VARCHAR}, @@ -314,6 +327,7 @@ name = #{record.name,jdbcType=VARCHAR}, biz_name = #{record.bizName,jdbcType=VARCHAR}, domain_id = #{record.domainId,jdbcType=BIGINT}, + alias = #{record.alias,jdbcType=VARCHAR}, viewer = #{record.viewer,jdbcType=VARCHAR}, view_org = #{record.viewOrg,jdbcType=VARCHAR}, admin = #{record.admin,jdbcType=VARCHAR}, @@ -339,6 +353,9 @@ domain_id = #{domainId,jdbcType=BIGINT}, + + alias = #{alias,jdbcType=VARCHAR}, + viewer = #{viewer,jdbcType=VARCHAR}, @@ -377,6 +394,7 @@ set name = #{name,jdbcType=VARCHAR}, biz_name = #{bizName,jdbcType=VARCHAR}, domain_id = #{domainId,jdbcType=BIGINT}, + alias = #{alias,jdbcType=VARCHAR}, viewer = #{viewer,jdbcType=VARCHAR}, view_org = #{viewOrg,jdbcType=VARCHAR}, admin = #{admin,jdbcType=VARCHAR}, @@ -394,6 +412,7 @@ set name = #{name,jdbcType=VARCHAR}, biz_name = #{bizName,jdbcType=VARCHAR}, domain_id = #{domainId,jdbcType=BIGINT}, + alias = #{alias,jdbcType=VARCHAR}, viewer = #{viewer,jdbcType=VARCHAR}, view_org = #{viewOrg,jdbcType=VARCHAR}, admin = #{admin,jdbcType=VARCHAR}, diff --git a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index 14f6c9db4..05cceb344 100644 --- a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -2,23 +2,28 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + @@ -102,5 +107,36 @@ + + diff --git a/semantic/model/src/main/resources/sql.ddl/s2_datasource.sql b/semantic/model/src/main/resources/sql.ddl/s2_datasource.sql index 2bb0d0c77..79bdadc17 100644 --- a/semantic/model/src/main/resources/sql.ddl/s2_datasource.sql +++ b/semantic/model/src/main/resources/sql.ddl/s2_datasource.sql @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` ( 100 ) NOT NULL COMMENT '更新人' - ) comment '数据源表' + ) comment '数据源表' \ No newline at end of file diff --git a/semantic/query/pom.xml b/semantic/query/pom.xml index 82f96e737..2f507ce5a 100644 --- a/semantic/query/pom.xml +++ b/semantic/query/pom.xml @@ -86,12 +86,7 @@ - - com.google.guava - guava - ${guava.version} - - \ No newline at end of file + diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java index 2d4455aff..7fb6f4107 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java @@ -79,7 +79,7 @@ public class QueryParser { } else { sql = sqlCommend.getSql(); for (String[] tb : tables) { - sql = sql.replaceAll(tb[0], "(" + tb[1] + ")"); + sql = StringUtils.replace(sql, tb[0], "(" + tb[1] + ")", -1); } } queryStatement.setSql(sql); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java index 8a7c63822..59f72a956 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.model.domain.Catalog; +import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DimensionTimeTypeParams; @@ -26,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -65,7 +67,7 @@ public class SemanticSchemaManager { Map> dimensionYamlTpls = new HashMap<>(); List datasourceYamlTpls = new ArrayList<>(); List metricYamlTpls = new ArrayList<>(); - catalog.getModelYamlTplByMoldelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls); + catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls); if (!datasourceYamlTpls.isEmpty()) { Map dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d)) .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); @@ -114,9 +116,19 @@ public class SemanticSchemaManager { datasource.setIdentifiers(getIdentify(d.getIdentifiers())); datasource.setDimensions(getDimensions(d.getDimensions())); datasource.setMeasures(getMeasures(d.getMeasures())); + datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions())); return datasource; } + private static String getDataSourceAggTime(List dimensions) { + Optional timeDimension = dimensions.stream() + .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst(); + if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) { + return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity(); + } + return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE; + } + private static List getMetricsByMetricYamlTpl(List metricYamlTpls) { List metrics = new ArrayList<>(); for (MetricYamlTpl metricYamlTpl : metricYamlTpls) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java index 5c9f755ac..300501f95 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java @@ -9,5 +9,8 @@ public class Constants { public static final String JOIN_TABLE_PREFIX = "src1_"; public static final String JOIN_TABLE_OUT_PREFIX = "src11_"; public static final String JOIN_TABLE_LEFT_PREFIX = "src12_"; + public static final String DIMENSION_TYPE_TIME_GRANULARITY_NONE = "none"; + public static final String DIMENSION_TYPE_TIME = "time"; + } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java index 9be8edde6..e250e0f70 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java @@ -20,4 +20,6 @@ public class DataSource { private List dimensions; private List measures; + + private String aggTime; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java index e4eb17197..8e6c35958 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java @@ -2,12 +2,13 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner; import com.tencent.supersonic.semantic.api.query.request.MetricReq; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode; +import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRender; import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender; @@ -16,6 +17,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; +import java.util.Objects; import java.util.Stack; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; @@ -42,6 +44,10 @@ public class AggPlanner implements Planner { if (datasource == null || datasource.isEmpty()) { throw new Exception("datasource not found"); } + if (Objects.nonNull(datasource.get(0).getAggTime()) && !datasource.get(0).getAggTime().equalsIgnoreCase( + Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { + isAgg = true; + } sourceId = String.valueOf(datasource.get(0).getSourceId()); // build level by level diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryReqConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java similarity index 69% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryReqConverter.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java index a276157b7..520377951 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryReqConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java @@ -1,19 +1,26 @@ -package com.tencent.supersonic.semantic.query.service; +package com.tencent.supersonic.semantic.query.parser.convert; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq; +import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptor; +import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptorFactory; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; +import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -21,6 +28,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @Component +@Slf4j public class QueryReqConverter { @Autowired @@ -30,14 +38,18 @@ public class QueryReqConverter { @Autowired private QueryStructUtils queryStructUtils; + @Autowired + private Catalog catalog; + public QueryStatement convert(QueryDslReq databaseReq, List domainSchemas) throws Exception { List tables = new ArrayList<>(); MetricTable metricTable = new MetricTable(); - String sql = databaseReq.getSql(); - List allFields = SqlParserSelectHelper.getAllFields(sql); - String tableName = SqlParserSelectHelper.getTableName(sql); + List allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql()); + String tableName = SqlParserSelectHelper.getTableName(databaseReq.getSql()); + + functionNameCorrector(databaseReq); if (CollectionUtils.isEmpty(domainSchemas) || StringUtils.isEmpty(tableName)) { return new QueryStatement(); @@ -56,7 +68,7 @@ public class QueryReqConverter { Set collect = allFields.stream().filter(entry -> dimensions.contains(entry.toLowerCase())) .map(String::toLowerCase).collect(Collectors.toSet()); for (String internalCol : QueryStructUtils.internalCols) { - if (sql.contains(internalCol)) { + if (databaseReq.getSql().contains(internalCol)) { collect.add(internalCol); } } @@ -80,4 +92,19 @@ public class QueryReqConverter { return queryStatement; } + private void functionNameCorrector(QueryDslReq databaseReq) { + DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId()); + if (Objects.isNull(database) || Objects.isNull(database.getType())) { + return; + } + String type = database.getType(); + EngineAdaptor engineAdaptor = EngineAdaptorFactory.getEngineAdaptor(type.toLowerCase()); + log.info("type:{},engineAdaptor:{}", type, engineAdaptor); + if (Objects.nonNull(engineAdaptor)) { + String functionNameCorrector = engineAdaptor.functionNameCorrector(databaseReq.getSql()); + log.info("sql:{} ,after corrector", databaseReq.getSql(), functionNameCorrector); + databaseReq.setSql(functionNameCorrector); + } + } + } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index d6a4d38fc..62e45b554 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.query.executor.QueryExecutor; +import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryUtils; import com.tencent.supersonic.semantic.query.utils.StatUtils; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java index 986275e74..36afdef4e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java @@ -24,7 +24,6 @@ import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -169,7 +168,7 @@ public class DataPermissionAOP { String modelName = modelResp.getName(); List admins = modelService.getModelAdmin(modelResp.getId()); String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins); - throw new InvalidArgumentException(message); + throw new InvalidPermissionException(message); } } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java index 7c45ae91c..f76fc2c97 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java @@ -97,6 +97,9 @@ public class QueryUtils { column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType()); column.setDataFormat(metricRespMap.get(nameEn).getDataFormat()); } + if (StringUtils.isEmpty(column.getShowType())) { + column.setShowType("NUMBER"); + } }); } @@ -147,6 +150,9 @@ public class QueryUtils { || type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) { return true; } + if (type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int")) { + return true; + } return false; }