diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a482288..fbc80da61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ - "Breaking Changes" describes any changes that may break existing functionality or cause compatibility issues with previous versions. +## SuperSonic [0.8.6] - 2024-02-23 + +### Added +- support view abstraction to Headless. +- add the Metric API to Headless and optimizing the Headless API. +- add integration tests to Headless. +- add TimeCorrector to Chat. + ## SuperSonic [0.8.4] - 2024-01-19 ### Added diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index 92434bf47..0d4678286 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -2,16 +2,14 @@ package com.tencent.supersonic.chat.api.pojo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import org.springframework.util.CollectionUtils; - import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import org.springframework.util.CollectionUtils; public class SemanticSchema implements Serializable { @@ -54,35 +52,6 @@ public class SemanticSchema implements Serializable { } } - public SchemaElement getElementByName(SchemaElementType elementType, String name) { - Optional element = Optional.empty(); - - switch (elementType) { - case ENTITY: - element = getElementsByNameOrAlias(name, getEntities()); - break; - case VIEW: - element = getElementsByNameOrAlias(name, getViews()); - break; - case METRIC: - element = getElementsByNameOrAlias(name, getMetrics()); - break; - case DIMENSION: - element = getElementsByNameOrAlias(name, getDimensions()); - break; - case VALUE: - element = getElementsByNameOrAlias(name, getDimensionValues()); - break; - default: - } - - if (element.isPresent()) { - return element.get(); - } else { - return null; - } - } - public Map getViewIdToName() { return viewSchemaList.stream() .collect(Collectors.toMap(a -> a.getView().getId(), a -> a.getView().getName(), (k1, k2) -> k1)); @@ -159,14 +128,6 @@ public class SemanticSchema implements Serializable { .findFirst(); } - private Optional getElementsByNameOrAlias(String name, List elements) { - return elements.stream() - .filter(schemaElement -> - name.equals(schemaElement.getName()) || (Objects.nonNull(schemaElement.getAlias()) - && schemaElement.getAlias().contains(name)) - ).findFirst(); - } - public SchemaElement getView(Long viewId) { List views = getViews(); return getElementsById(viewId, views).orElse(null); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java index f0782f85c..0059a8eb6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java @@ -73,9 +73,6 @@ public class OptimizationConfig { @Value("${text2sql.self.consistency.num:5}") private int text2sqlSelfConsistencyNum; - @Value("${text2sql.collection.name:text2dsl_agent_collection}") - private String text2sqlCollectionName; - @Value("${parse.show.count:3}") private Integer parseShowCount; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java index 48011fe23..09a8beaa3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.chat.core.corrector; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; @@ -12,6 +13,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.springframework.core.env.Environment; import org.springframework.util.CollectionUtils; import java.util.ArrayList; @@ -77,7 +79,13 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { protected void addFieldsToSelect(SemanticParseInfo semanticParseInfo, String correctS2SQL) { Set selectFields = new HashSet<>(SqlSelectHelper.getSelectFields(correctS2SQL)); Set needAddFields = new HashSet<>(SqlSelectHelper.getGroupByFields(correctS2SQL)); - //needAddFields.addAll(SqlSelectHelper.getOrderByFields(correctS2SQL)); + + //decide whether add order by expression field to select + Environment environment = ContextUtils.getBean(Environment.class); + String correctorAdditionalInfo = environment.getProperty("corrector.additional.information"); + if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) { + needAddFields.addAll(SqlSelectHelper.getOrderByFields(correctS2SQL)); + } // If there is no aggregate function in the S2SQL statement and // there is a data field in 'WHERE' statement, add the field to the 'SELECT' statement. diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java index fb8172ba7..a88c9c945 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java @@ -45,7 +45,8 @@ public class GroupByCorrector extends BaseSemanticCorrector { ViewResp viewResp = viewService.getView(viewId); List modelIds = viewResp.getViewDetail().getViewModelConfigs().stream().map(config -> config.getId()) .collect(Collectors.toList()); - MetaFilter metaFilter = new MetaFilter(modelIds); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(modelIds); List modelRespList = modelService.getModelList(metaFilter); for (ModelResp modelResp : modelRespList) { List dimList = modelResp.getModelDetail().getDimensions(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java index ba4e48e0d..e09d68ce2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java @@ -3,11 +3,14 @@ package com.tencent.supersonic.chat.core.corrector; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.env.Environment; import org.springframework.util.CollectionUtils; import java.util.List; @@ -26,8 +29,12 @@ public class HavingCorrector extends BaseSemanticCorrector { //add aggregate to all metric addHaving(queryContext, semanticParseInfo); - //add having expression filed to select - //addHavingToSelect(semanticParseInfo); + //decide whether add having expression field to select + Environment environment = ContextUtils.getBean(Environment.class); + String correctorAdditionalInfo = environment.getProperty("corrector.additional.information"); + if (StringUtils.isNotBlank(correctorAdditionalInfo) && Boolean.parseBoolean(correctorAdditionalInfo)) { + addHavingToSelect(semanticParseInfo); + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java index 7c471f320..07ef29067 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.core.parser.sql.llm.S2SqlDateHelper; +import com.tencent.supersonic.chat.core.utils.S2SqlDateHelper; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; @@ -18,6 +18,7 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; @@ -65,11 +66,20 @@ public class WhereCorrector extends BaseSemanticCorrector { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) { - String currentDate = S2SqlDateHelper.getReferenceDate(queryContext, semanticParseInfo.getViewId()); - if (StringUtils.isNotBlank(currentDate)) { + Pair startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, + semanticParseInfo.getViewId(), semanticParseInfo.getQueryType()); + if (StringUtils.isNotBlank(startEndDate.getLeft()) + && StringUtils.isNotBlank(startEndDate.getRight())) { correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); - correctS2SQL = SqlAddHelper.addWhere( - correctS2SQL, TimeDimensionEnum.DAY.getChName(), currentDate); + String dateChName = TimeDimensionEnum.DAY.getChName(); + String condExpr = String.format(" ( %s >= '%s' and %s <= '%s' )", dateChName, + startEndDate.getLeft(), dateChName, startEndDate.getRight()); + try { + Expression expression = CCJSqlParserUtil.parseCondExpression(condExpr); + correctS2SQL = SqlAddHelper.addWhere(correctS2SQL, expression); + } catch (JSQLParserException e) { + log.error("parseCondExpression:{}", e); + } } } semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java index 296d951f8..00d4a7b5b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java @@ -10,9 +10,10 @@ import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult; import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.server.service.KnowledgeService; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; /*** * A mapper that recognizes schema elements with vector embedding. @@ -24,7 +25,8 @@ public class EmbeddingMapper extends BaseMapper { public void doMap(QueryContext queryContext) { //1. query from embedding by queryText String queryText = queryContext.getQueryText(); - List terms = HanlpHelper.getTerms(queryText); + KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class); + List terms = knowledgeService.getTerms(queryText); EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class); List matchResults = matchStrategy.getMatches(queryContext, terms); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java index 3dce92fb3..cde924968 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.core.utils.S2SqlDateHelper; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java index 13eeee5fa..1342bf0cd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java @@ -48,7 +48,7 @@ public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), - optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); + optimizationConfig.getText2sqlExampleNum()); List>> exampleListPool = sqlPromptGenerator.getExampleCombos(sqlExamples, optimizationConfig.getText2sqlFewShotsNum(), optimizationConfig.getText2sqlSelfConsistencyNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java index a3fe2748f..a98486ae1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java @@ -45,7 +45,7 @@ public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { //1.retriever sqlExamples keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), - optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); + optimizationConfig.getText2sqlExampleNum()); //2.generator linking and sql prompt by sqlExamples,and generate response. String promptStr = sqlPromptGenerator.generatorLinkingAndSqlPrompt(llmReq, sqlExamples); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java deleted file mode 100644 index 61a51f49d..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; - -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.common.util.DatePeriodEnum; -import com.tencent.supersonic.common.util.DateUtils; -import java.util.Objects; - -public class S2SqlDateHelper { - - public static String getReferenceDate(QueryContext queryContext, Long modelId) { - String defaultDate = DateUtils.getBeforeDate(0); - if (Objects.isNull(modelId)) { - return defaultDate; - } - ChatConfigFilter filter = new ChatConfigFilter(); - filter.setModelId(modelId); - ChatConfigRichResp chatConfigRichResp = queryContext.getModelIdToChatRichConfig().get(modelId); - - if (Objects.isNull(chatConfigRichResp)) { - return defaultDate; - } - if (Objects.isNull(chatConfigRichResp.getChatDetailRichConfig()) || Objects.isNull( - chatConfigRichResp.getChatDetailRichConfig().getChatDefaultConfig())) { - return defaultDate; - } - - ChatDefaultRichConfigResp chatDefaultConfig = chatConfigRichResp.getChatDetailRichConfig() - .getChatDefaultConfig(); - Integer unit = chatDefaultConfig.getUnit(); - String period = chatDefaultConfig.getPeriod(); - if (Objects.nonNull(unit)) { - // If the unit is set to less than 0, then do not add relative date. - if (unit < 0) { - return null; - } - DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(period); - if (Objects.isNull(datePeriodEnum)) { - return DateUtils.getBeforeDate(unit); - } else { - return DateUtils.getBeforeDate(unit, datePeriodEnum); - } - } - return defaultDate; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java index 7aa5c869a..7a586e9d3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.ComponentFactory; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; @@ -19,6 +20,7 @@ import java.util.Objects; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.stereotype.Component; @@ -32,6 +34,9 @@ public class SqlExamplarLoader { private TypeReference> valueTypeRef = new TypeReference>() { }; + @Autowired + private EmbeddingConfig embeddingConfig; + public List getSqlExamples() throws IOException { ClassPathResource resource = new ClassPathResource(EXAMPLE_JSON_FILE); InputStream inputStream = resource.getInputStream(); @@ -53,8 +58,8 @@ public class SqlExamplarLoader { s2EmbeddingStore.addQuery(collectionName, queries); } - public List> retrieverSqlExamples(String queryText, String collectionName, int maxResults) { - + public List> retrieverSqlExamples(String queryText, int maxResults) { + String collectionName = embeddingConfig.getText2sqlCollectionName(); RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(Collections.singletonList(queryText)) .queryEmbeddings(null).build(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java index 3ba1452b5..a90af3086 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java @@ -44,7 +44,7 @@ public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { //1.retriever sqlExamples and generate exampleListPool keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), - optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); + optimizationConfig.getText2sqlExampleNum()); List>> exampleListPool = sqlPromptGenerator.getExampleCombos(sqlExamples, optimizationConfig.getText2sqlFewShotsNum(), optimizationConfig.getText2sqlSelfConsistencyNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java index aaaf3eb68..61fe24b60 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java @@ -43,7 +43,7 @@ public class TwoPassSqlGeneration implements SqlGeneration, InitializingBean { public LLMResp generation(LLMReq llmReq, Long viewId) { keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), - optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); + optimizationConfig.getText2sqlExampleNum()); String linkingPromptStr = sqlPromptGenerator.generateLinkingPrompt(llmReq, sqlExamples); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 03fc03164..3a38c49b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -121,7 +121,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } QueryStructReq queryStructReq = convertQueryStruct(); convertBizNameToName(semanticSchema, queryStructReq); - QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySQLReq = queryStructReq.convert(); parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql()); parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java index 44d9522a8..1bbc151f6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java @@ -44,6 +44,7 @@ public class ViewSchemaBuilder { SchemaElement metricToAdd = SchemaElement.builder() .view(resp.getId()) + .model(metric.getModelId()) .id(metric.getId()) .name(metric.getName()) .bizName(metric.getBizName()) @@ -84,6 +85,7 @@ public class ViewSchemaBuilder { } SchemaElement dimToAdd = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -96,6 +98,7 @@ public class ViewSchemaBuilder { SchemaElement dimValueToAdd = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -107,6 +110,7 @@ public class ViewSchemaBuilder { if (dim.getIsTag() == 1) { SchemaElement tagToAdd = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -126,6 +130,7 @@ public class ViewSchemaBuilder { if (dim != null) { SchemaElement entity = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java new file mode 100644 index 000000000..6da417634 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java @@ -0,0 +1,68 @@ +package com.tencent.supersonic.chat.core.utils; + +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.common.util.DatePeriodEnum; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import java.util.Objects; +import org.apache.commons.lang3.tuple.Pair; + +public class S2SqlDateHelper { + + public static String getReferenceDate(QueryContext queryContext, Long viewId) { + String defaultDate = DateUtils.getBeforeDate(0); + if (Objects.isNull(viewId)) { + return defaultDate; + } + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); + if (viewSchema == null || viewSchema.getTagTypeTimeDefaultConfig() == null) { + return defaultDate; + } + TimeDefaultConfig tagTypeTimeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + return getDefaultDate(defaultDate, tagTypeTimeDefaultConfig).getLeft(); + } + + public static Pair getStartEndDate(QueryContext queryContext, + Long viewId, QueryType queryType) { + String defaultDate = DateUtils.getBeforeDate(0); + if (Objects.isNull(viewId)) { + return Pair.of(defaultDate, defaultDate); + } + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); + if (viewSchema == null) { + return Pair.of(defaultDate, defaultDate); + } + TimeDefaultConfig defaultConfig = viewSchema.getMetricTypeTimeDefaultConfig(); + if (QueryType.TAG.equals(queryType)) { + defaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + } + return getDefaultDate(defaultDate, defaultConfig); + } + + private static Pair getDefaultDate(String defaultDate, TimeDefaultConfig defaultConfig) { + if (Objects.isNull(defaultConfig)) { + return Pair.of(null, null); + } + Integer unit = defaultConfig.getUnit(); + String period = defaultConfig.getPeriod(); + TimeMode timeMode = defaultConfig.getTimeMode(); + if (Objects.nonNull(unit)) { + // If the unit is set to less than 0, then do not add relative date. + if (unit < 0) { + return Pair.of(null, null); + } + DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(period); + String startDate = DateUtils.getBeforeDate(unit, datePeriodEnum); + String endDate = DateUtils.getBeforeDate(1, datePeriodEnum); + if (TimeMode.LAST.equals(timeMode)) { + endDate = startDate; + } + return Pair.of(startDate, endDate); + } + return Pair.of(defaultDate, defaultDate); + } + +} diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java new file mode 100644 index 000000000..54fd38e4b --- /dev/null +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java @@ -0,0 +1,121 @@ +package com.tencent.supersonic.chat.core.utils; + + +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +class S2SqlDateHelperTest { + + @Test + void getReferenceDate() { + Long viewId = 1L; + QueryContext queryContext = buildQueryContext(viewId); + + String referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, null); + Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(0)); + + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(0)); + + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); + QueryConfig queryConfig = viewSchema.getQueryConfig(); + TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); + timeDefaultConfig.setTimeMode(TimeMode.LAST); + timeDefaultConfig.setPeriod(Constants.DAY); + timeDefaultConfig.setUnit(20); + queryConfig.getTagTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); + + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(20)); + + timeDefaultConfig.setUnit(1); + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + Assert.assertEquals(referenceDate, DateUtils.getBeforeDate(1)); + + timeDefaultConfig.setUnit(-1); + referenceDate = S2SqlDateHelper.getReferenceDate(queryContext, viewId); + Assert.assertNull(referenceDate); + } + + @Test + void getStartEndDate() { + Long viewId = 1L; + QueryContext queryContext = buildQueryContext(viewId); + + Pair startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, null, QueryType.TAG); + Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(0)); + Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(0)); + + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.TAG); + Assert.assertNull(startEndDate.getLeft()); + Assert.assertNull(startEndDate.getRight()); + + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); + QueryConfig queryConfig = viewSchema.getQueryConfig(); + TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); + timeDefaultConfig.setTimeMode(TimeMode.LAST); + timeDefaultConfig.setPeriod(Constants.DAY); + timeDefaultConfig.setUnit(20); + queryConfig.getTagTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); + queryConfig.getMetricTypeDefaultConfig().setTimeDefaultConfig(timeDefaultConfig); + + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.TAG); + Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(20)); + Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(20)); + + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(20)); + Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(20)); + + timeDefaultConfig.setUnit(2); + timeDefaultConfig.setTimeMode(TimeMode.RECENT); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(2)); + Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(1)); + + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.TAG); + Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(2)); + Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(1)); + + timeDefaultConfig.setUnit(-1); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + Assert.assertNull(startEndDate.getLeft()); + Assert.assertNull(startEndDate.getRight()); + + timeDefaultConfig.setTimeMode(TimeMode.LAST); + timeDefaultConfig.setPeriod(Constants.DAY); + timeDefaultConfig.setUnit(5); + startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, viewId, QueryType.METRIC); + Assert.assertEquals(startEndDate.getLeft(), DateUtils.getBeforeDate(5)); + Assert.assertEquals(startEndDate.getRight(), DateUtils.getBeforeDate(5)); + } + + private QueryContext buildQueryContext(Long viewId) { + QueryContext queryContext = new QueryContext(); + List viewSchemaList = new ArrayList<>(); + ViewSchema viewSchema = new ViewSchema(); + QueryConfig queryConfig = new QueryConfig(); + viewSchema.setQueryConfig(queryConfig); + SchemaElement schemaElement = new SchemaElement(); + schemaElement.setView(viewId); + viewSchema.setView(schemaElement); + viewSchemaList.add(viewSchema); + + SemanticSchema semanticSchema = new SemanticSchema(viewSchemaList); + queryContext.setSemanticSchema(semanticSchema); + return queryContext; + } +} \ No newline at end of file diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index 84035877f..5c599c81e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -38,11 +38,11 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { queryResult.setRecommendedDimensions(dimensionRecommended); } - private List getDimensions(Long metricId, Long modelId) { + private List getDimensions(Long metricId, Long viewId) { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ViewSchema modelSchema = semanticService.getModelSchema(modelId); + ViewSchema viewSchema = semanticService.getViewSchema(viewId); List drillDownDimensions = Lists.newArrayList(); - Set metricElements = modelSchema.getMetrics(); + Set metricElements = viewSchema.getMetrics(); if (!CollectionUtils.isEmpty(metricElements)) { Optional metric = metricElements.stream().filter(schemaElement -> metricId.equals(schemaElement.getId()) @@ -54,7 +54,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { } } final List drillDownDimensionsFinal = drillDownDimensions; - return modelSchema.getDimensions().stream() + return viewSchema.getDimensions().stream() .filter(dim -> filterDimension(drillDownDimensionsFinal, dim)) .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(recommend_dimension_size) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java index 3b5e5f60f..ec9423da2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java @@ -50,7 +50,7 @@ public class SemanticService { return schemaService.getSemanticSchema(); } - public ViewSchema getModelSchema(Long id) { + public ViewSchema getViewSchema(Long id) { return schemaService.getViewSchema(id); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 5fb1c43ae..d3b6b4cc2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -219,7 +219,7 @@ public class ConfigServiceImpl implements ConfigService { } BeanUtils.copyProperties(chatConfigResp, chatConfigRich); - ViewSchema viewSchema = semanticService.getModelSchema(modelId); + ViewSchema viewSchema = semanticService.getViewSchema(modelId); if (viewSchema == null) { return chatConfigRich; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java index fd9cf56e3..9d76b7bca 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java @@ -48,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService { if (Objects.isNull(modelId)) { return new RecommendResp(); } - ViewSchema modelSchema = semanticService.getModelSchema(modelId); + ViewSchema modelSchema = semanticService.getViewSchema(modelId); if (Objects.isNull(modelSchema)) { return new RecommendResp(); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java index fce843828..14e2c5b90 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java @@ -1,17 +1,19 @@ package com.tencent.supersonic.chat.server.service.impl; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.core.knowledge.DictWord; -import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory; import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; @Service @@ -28,7 +30,6 @@ public class WordService { addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words); addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words); - addWordsByType(DictWordType.VIEW, semanticSchema.getViews(), words); addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words); addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words); @@ -36,6 +37,7 @@ public class WordService { } private void addWordsByType(DictWordType value, List metas, List natures) { + metas = distinct(metas); List natureList = WordBuilderFactory.get(value).getDictWords(metas); log.debug("nature type:{} , nature size:{}", value.name(), natureList.size()); natures.addAll(natureList); @@ -48,4 +50,13 @@ public class WordService { public void setPreDictWords(List preDictWords) { this.preDictWords = preDictWords; } + + private List distinct(List metas) { + return metas.stream() + .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1)) + .values() + .stream() + .collect(Collectors.toList()); + } + } diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java index 6a6e222f6..59e3dc51d 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java @@ -52,14 +52,14 @@ class QueryReqBuilderTest { orders.add(order); queryStructReq.setOrders(orders); - QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySQLReq = queryStructReq.convert(); Assert.assertEquals( "SELECT department, SUM(pv) AS pv FROM 内容库 " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " + "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); queryStructReq.setQueryType(QueryType.TAG); - querySQLReq = queryStructReq.convert(queryStructReq); + querySQLReq = queryStructReq.convert(); Assert.assertEquals( "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "ORDER BY uv LIMIT 2000", diff --git a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java index b28d2931d..c8b9d6f9b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java @@ -32,6 +32,9 @@ public class EmbeddingConfig { @Value("${embedding.metric.analyzeQuery.collection:solved_query_collection}") private String metricAnalyzeQueryCollection; + @Value("${text2sql.collection.name:text2dsl_agent_collection}") + private String text2sqlCollectionName; + @Value("${embedding.metric.analyzeQuery.nResult:5}") private int metricAnalyzeQueryResultNum; diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java index ca1c46e69..105d82418 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java @@ -68,6 +68,9 @@ public class DateUtils { } public static String getBeforeDate(int intervalDay, DatePeriodEnum datePeriodEnum) { + if (Objects.isNull(datePeriodEnum)) { + return getBeforeDate(intervalDay); + } SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); String currentDate = dateFormat.format(new Date()); return getBeforeDate(currentDate, intervalDay, datePeriodEnum); @@ -101,7 +104,7 @@ public class DateUtils { int month = tempDate.get(ChronoField.MONTH_OF_YEAR); int firstMonthOfQuarter = ((month - 1) / 3) * 3 + 1; return tempDate.with(ChronoField.MONTH_OF_YEAR, firstMonthOfQuarter) - .with(TemporalAdjusters.firstDayOfMonth()); + .with(TemporalAdjusters.firstDayOfMonth()); }; result = result.with(firstDayOfQuarter); } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/InMemoryS2EmbeddingStore.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/InMemoryS2EmbeddingStore.java index d254e0637..6c063e19d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/embedding/InMemoryS2EmbeddingStore.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/InMemoryS2EmbeddingStore.java @@ -48,7 +48,9 @@ public class InMemoryS2EmbeddingStore implements S2EmbeddingStore { InMemoryEmbeddingStore embeddingStore = null; Path filePath = getPersistentPath(collectionName); try { - if (Files.exists(filePath)) { + EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); + if (Files.exists(filePath) && !collectionName.equals(embeddingConfig.getMetaCollectionName()) + && !collectionName.equals(embeddingConfig.getText2sqlCollectionName())) { embeddingStore = InMemoryEmbeddingStore.fromFile(filePath); embeddingStore.entries = new CopyOnWriteArraySet<>(embeddingStore.entries); log.info("embeddingStore reload from file:{}", filePath); diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java index f1596d3d8..13343691a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java @@ -1,5 +1,11 @@ package com.tencent.supersonic.common.util.jsqlparser; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Alias; @@ -9,6 +15,7 @@ import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; import net.sf.jsqlparser.expression.Function; import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.Parenthesis; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.WhenClause; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; @@ -33,13 +40,6 @@ import net.sf.jsqlparser.statement.select.SubSelect; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - /** * Sql Parser Select Helper */ @@ -543,6 +543,10 @@ public class SqlSelectHelper { getColumnFromExpr(expr.getLeftExpression(), columns); getColumnFromExpr(expr.getRightExpression(), columns); } + if (expression instanceof Parenthesis) { + Parenthesis expr = (Parenthesis) expression; + getColumnFromExpr(expr.getExpression(), columns); + } } } diff --git a/evaluation/build_pred_result.py b/evaluation/build_pred_result.py index e5ef4c18f..d21324b28 100644 --- a/evaluation/build_pred_result.py +++ b/evaluation/build_pred_result.py @@ -65,12 +65,19 @@ def get_pred_result(): questions=read_query(input_path) pred_sql_list=[] default_sql="select * from tablea " + time_cost=[] for i in range(0,len(questions)): + start_time = time.time() pred_sql=get_pred_sql(questions[i],url,agent_id,chat_id,authorization,default_sql) + end_time = time.time() + cost='%.3f'%(end_time-start_time) + time_cost.append(cost) pred_sql_list.append(pred_sql) time.sleep(60) write_sql(pred_sql_path, pred_sql_list) + return [float(cost) for cost in time_cost] + if __name__ == "__main__": print("pred") diff --git a/evaluation/evaluation.py b/evaluation/evaluation.py index ae2ed1cfb..1e0246579 100644 --- a/evaluation/evaluation.py +++ b/evaluation/evaluation.py @@ -482,7 +482,7 @@ def print_scores(scores, etype): print("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(type_, *this_scores)) -def evaluate(gold, predict, db_dir, etype, kmaps,query_path): +def evaluate(gold, predict, db_dir, etype, kmaps,query_path,time_cost): with open(gold) as f: glist = [l.strip().split('\t') for l in f.readlines() if len(l.strip()) > 0] @@ -597,7 +597,11 @@ def evaluate(gold, predict, db_dir, etype, kmaps,query_path): scores[level]['partial'][type_]['f1'] = \ 2.0 * scores[level]['partial'][type_]['acc'] * scores[level]['partial'][type_]['rec'] / ( scores[level]['partial'][type_]['rec'] + scores[level]['partial'][type_]['acc']) - + cost_dic = {} + cost_dic["max_time"] = max(time_cost) + cost_dic["min_time"] = min(time_cost) + cost_dic["avg_time"] = sum(time_cost)/len(time_cost) + log_list.append(cost_dic) print_scores(scores, etype) print(scores['all']['exec']) current_directory = os.path.dirname(os.path.abspath(__file__)) @@ -608,7 +612,6 @@ def evaluate(gold, predict, db_dir, etype, kmaps,query_path): with open(file_name, 'w') as json_file: json.dump(log_list, json_file, indent=4, ensure_ascii=False) - def eval_exec_match(db, p_str, g_str, pred, gold): """ return 1 if the values between prediction and gold are matching @@ -890,7 +893,7 @@ def build_foreign_key_map_from_json(table): tables[entry['db_id']] = build_foreign_key_map(entry) return tables -def get_evaluation_result(): +def get_evaluation_result(time_cost): current_directory = os.path.dirname(os.path.abspath(__file__)) config_file=current_directory+"/config/config.yaml" with open(config_file, 'r') as file: @@ -905,7 +908,7 @@ def get_evaluation_result(): etype="exec" kmaps = build_foreign_key_map_from_json(table) - evaluate(gold, pred, db_dir, etype, kmaps,query_path) + evaluate(gold, pred, db_dir, etype, kmaps,query_path,time_cost) def remove_unused_file(): current_directory = os.path.dirname(os.path.abspath(__file__)) @@ -927,8 +930,8 @@ def remove_unused_file(): if __name__ == "__main__": build_table() - get_pred_result() - get_evaluation_result() + time_cost=get_pred_result() + get_evaluation_result(time_cost) remove_unused_file() diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index ba2960953..28b7c1092 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -19,13 +19,15 @@ public class ModelDetail { private String tableQuery; - private List identifiers; + private List identifiers = Lists.newArrayList(); - private List dimensions; + private List dimensions = Lists.newArrayList(); - private List measures; + private List measures = Lists.newArrayList(); - private List fields; + private List fields = Lists.newArrayList(); + + private List sqlVariables = Lists.newArrayList(); public String getSqlQuery() { if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java index 6dec090f5..dbf36dce5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo; import lombok.Data; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; @Data public class Param { @@ -10,7 +11,7 @@ public class Param { @NotBlank(message = "Invald parameter name") private String name; - @NotBlank(message = "Invalid parameter value") + @NotNull(message = "Invalid parameter value") private String value; public Param() { @@ -21,14 +22,4 @@ public class Param { this.value = value; } - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("{"); - sb.append("\"name\":\"") - .append(name).append('\"'); - sb.append(",\"value\":\"") - .append(value).append('\"'); - sb.append('}'); - return sb.toString(); - } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java index 22e50df9e..f80f5cda5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.base.Objects; -import com.google.common.collect.Lists; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -19,6 +18,7 @@ import java.util.List; public class SchemaElement implements Serializable { private Long view; + private Long model; private Long id; private String name; private String bizName; @@ -52,7 +52,4 @@ public class SchemaElement implements Serializable { return Objects.hashCode(view, id, name, bizName, type); } - public List getModelNames() { - return Lists.newArrayList(name); - } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java new file mode 100644 index 000000000..bdee9e5dd --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlVariable.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.api.pojo; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.enums.VariableValueType; +import lombok.Data; + +import java.util.List; + +@Data +public class SqlVariable { + private String name; + private VariableValueType valueType; + private List defaultValues = Lists.newArrayList(); +} + + diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/VariableValueType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/VariableValueType.java new file mode 100644 index 000000000..59fdd33fc --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/VariableValueType.java @@ -0,0 +1,7 @@ +package com.tencent.supersonic.headless.api.pojo.enums; + +public enum VariableValueType { + STRING, + NUMBER, + EXPR +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java index 4ba3b6dbc..7e9c11459 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.request; +import com.tencent.supersonic.common.pojo.DateConf; import java.util.List; import lombok.Data; import lombok.ToString; @@ -18,4 +19,8 @@ public class QueryMetricReq { private List dimensionNames; + private DateConf dateInfo = new DateConf(); + + private Long limit = 2000L; + } \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 679563c9d..470a2afdf 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -44,6 +44,7 @@ import java.util.stream.Collectors; @Data @Slf4j public class QueryStructReq extends SemanticQueryReq { + private List groups = new ArrayList<>(); private List aggregators = new ArrayList<>(); private List orders = new ArrayList<>(); @@ -151,28 +152,27 @@ public class QueryStructReq extends SemanticQueryReq { return sb.toString(); } - public QuerySqlReq convert(QueryStructReq queryStructReq) { - return convert(queryStructReq, false); + public QuerySqlReq convert() { + return convert(false); } /** * convert queryStructReq to QueryS2SQLReq * - * @param queryStructReq * @return */ - public QuerySqlReq convert(QueryStructReq queryStructReq, boolean isBizName) { + public QuerySqlReq convert(boolean isBizName) { String sql = null; try { - sql = buildSql(queryStructReq, isBizName); + sql = buildSql(this, isBizName); } catch (Exception e) { log.error("buildSql error", e); } QuerySqlReq result = new QuerySqlReq(); result.setSql(sql); - result.setViewId(queryStructReq.getViewId()); - result.setModelIds(queryStructReq.getModelIdSet()); + result.setViewId(this.getViewId()); + result.setModelIds(this.getModelIdSet()); result.setParams(new ArrayList<>()); return result; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java index 09e4fb81a..c59a15293 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java @@ -16,8 +16,6 @@ public class ViewReq extends SchemaItem { private String alias; - private String filterSql; - private QueryConfig queryConfig; private List admins; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java index 8f0161a3d..e92752ec1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java @@ -30,6 +30,8 @@ public class MetricResp extends SchemaItem { private Long domainId; + private String modelBizName; + private String modelName; //ATOMIC DERIVED diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java index f96617aeb..ac515e6e9 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java @@ -1,12 +1,14 @@ package com.tencent.supersonic.headless.api.pojo.response; +import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.ModelRela; -import com.tencent.supersonic.headless.api.pojo.Identify; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - -import java.util.List; +import org.apache.commons.collections4.CollectionUtils; @Data @AllArgsConstructor @@ -17,18 +19,17 @@ public class ModelSchemaResp extends ModelResp { private List dimensions; private List modelRelas; - public DimSchemaResp getPrimaryKey() { - Identify identify = getPrimaryIdentify(); - if (identify == null) { - return null; + public Set getModelClusterSet() { + if (CollectionUtils.isEmpty(this.modelRelas)) { + return Sets.newHashSet(); + } else { + Set modelClusterSet = new HashSet(); + this.modelRelas.forEach((modelRela) -> { + modelClusterSet.add(modelRela.getToModelId()); + modelClusterSet.add(modelRela.getFromModelId()); + }); + return modelClusterSet; } - for (DimSchemaResp dimension : dimensions) { - if (identify.getBizName().equals(dimension.getBizName())) { - dimension.setEntityAlias(identify.getEntityNames()); - return dimension; - } - } - return null; } } \ No newline at end of file diff --git a/headless/core/pom.xml b/headless/core/pom.xml index be65413f8..26a5e8e25 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -24,6 +24,11 @@ ${lombok.version} provided + + org.antlr + ST4 + ${st.version} + org.springframework @@ -106,6 +111,28 @@ org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + * + + + javax.servlet + servlet-api + + org.apache.commons diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java index 61809f67e..7e75f1ffb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java @@ -6,8 +6,15 @@ import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -17,11 +24,6 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; -import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - @Slf4j public class SearchService { @@ -39,14 +41,14 @@ public class SearchService { * @param key * @return */ - public static List prefixSearch(String key, int limit, Set detectModelIds) { - return prefixSearch(key, limit, trie, detectModelIds); + public static List prefixSearch(String key, int limit, Map> modelIdToViewIds) { + return prefixSearch(key, limit, trie, modelIdToViewIds); } public static List prefixSearch(String key, int limit, BinTrie> binTrie, - Set detectModelIds) { - Set>> result = prefixSearchLimit(key, limit, binTrie, detectModelIds); - return result.stream().map( + Map> modelIdToViewIds) { + Set>> result = prefixSearchLimit(key, limit, binTrie, modelIdToViewIds.keySet()); + List hanlpMapResults = result.stream().map( entry -> { String name = entry.getKey().replace("#", " "); return new HanlpMapResult(name, entry.getValue(), key); @@ -54,6 +56,13 @@ public class SearchService { ).sorted((a, b) -> -(b.getName().length() - a.getName().length())) .limit(SEARCH_SIZE) .collect(Collectors.toList()); + for (HanlpMapResult hanlpMapResult : hanlpMapResults) { + List natures = hanlpMapResult.getNatures().stream() + .map(nature -> NatureHelper.changeModel2View(nature, modelIdToViewIds)) + .flatMap(Collection::stream).collect(Collectors.toList()); + hanlpMapResult.setNatures(natures); + } + return hanlpMapResults; } /*** diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java index 96247a048..9d662fab1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java @@ -38,11 +38,11 @@ public class DimensionWordBuilder extends BaseWordBuilder { private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); dictWord.setWord(word); - Long viewId = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.DIMENSION.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.SUFFIX.getType() + DictWordType.DIMENSION.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java index 8ce190e2c..145eb29c1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java @@ -27,8 +27,8 @@ public class EntityWordBuilder extends BaseWordBuilder { return result; } - Long view = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + view + DictWordType.NATURE_SPILT + schemaElement.getId() + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getType(); if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java index c54fca038..fe6af5ef6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java @@ -38,11 +38,11 @@ public class MetricWordBuilder extends BaseWordBuilder { private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); dictWord.setWord(word); - Long viewId = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.METRIC.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java index 73f1bbbc0..95bbb39b0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java @@ -27,8 +27,8 @@ public class ValueWordBuilder extends BaseWordBuilder { schemaElement.getAlias().stream().forEach(value -> { DictWord dictWord = new DictWord(); - Long viewId = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId(); + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId(); dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); dictWord.setWord(value); result.add(dictWord); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java index 2883babce..c4a5b510d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java @@ -1,34 +1,36 @@ package com.tencent.supersonic.headless.core.knowledge.helper; -import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; - +import com.google.common.collect.Lists; import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.seg.Segment; +import com.hankcs.hanlp.seg.common.Term; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.headless.core.knowledge.HadoopFileIOAdapter; import com.tencent.supersonic.headless.core.knowledge.MapResult; import com.tencent.supersonic.headless.core.knowledge.MultiCustomDictionary; import com.tencent.supersonic.headless.core.knowledge.SearchService; -import com.tencent.supersonic.common.pojo.enums.DictWordType; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ResourceUtils; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; -import com.hankcs.hanlp.seg.common.Term; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; -import org.springframework.util.ResourceUtils; +import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; /** * HanLP helper @@ -212,18 +214,25 @@ public class HanlpHelper { } } - public static List getTerms(String text) { + public static List getTerms(String text, Map> modelIdToViewIds) { return getSegment().seg(text.toLowerCase()).stream() .filter(term -> term.getNature().startsWith(DictWordType.NATURE_SPILT)) - .map(term -> transform2ApiTerm(term)) + .map(term -> transform2ApiTerm(term, modelIdToViewIds)) + .flatMap(Collection::stream) .collect(Collectors.toList()); } - public static S2Term transform2ApiTerm(Term term) { - S2Term knowledgeTerm = new S2Term(); - BeanUtils.copyProperties(term, knowledgeTerm); - knowledgeTerm.setFrequency(term.getFrequency()); - return knowledgeTerm; + public static List transform2ApiTerm(Term term, Map> modelIdToViewIds) { + List s2Terms = Lists.newArrayList(); + List natures = NatureHelper.changeModel2View(String.valueOf(term.getNature()), modelIdToViewIds); + for (String nature : natures) { + S2Term s2Term = new S2Term(); + BeanUtils.copyProperties(term, s2Term); + s2Term.setNature(Nature.create(nature)); + s2Term.setFrequency(term.getFrequency()); + s2Terms.add(s2Term); + } + return s2Terms; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java index a37d35b92..7f88d626b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java @@ -1,12 +1,14 @@ package com.tencent.supersonic.headless.core.knowledge.helper; +import com.google.common.collect.Lists; import com.hankcs.hanlp.corpus.tag.Nature; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat; -import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Comparator; @@ -81,6 +83,43 @@ public class NatureHelper { return null; } + private static Long getModelId(String nature) { + try { + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length <= 1) { + return null; + } + return Long.valueOf(split[1]); + } catch (NumberFormatException e) { + log.error("", e); + } + return null; + } + + private static Nature changeModel2View(String nature, Long viewId) { + try { + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length <= 1) { + return null; + } + split[1] = String.valueOf(viewId); + return Nature.create(StringUtils.join(split, DictWordType.NATURE_SPILT)); + } catch (NumberFormatException e) { + log.error("", e); + } + return null; + } + + public static List changeModel2View(String nature, Map> modelIdToViewIds) { + Long modelId = getModelId(nature); + List viewIds = modelIdToViewIds.get(modelId); + if (CollectionUtils.isEmpty(viewIds)) { + return Lists.newArrayList(); + } + return viewIds.stream().map(viewId -> String.valueOf(changeModel2View(nature, viewId))) + .collect(Collectors.toList()); + } + public static boolean isDimensionValueViewId(String nature) { if (StringUtils.isEmpty(nature)) { return false; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java new file mode 100644 index 000000000..c1db3ac93 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java @@ -0,0 +1,120 @@ +package com.tencent.supersonic.headless.core.utils; + +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; +import com.tencent.supersonic.headless.api.pojo.Param; +import com.tencent.supersonic.headless.api.pojo.SqlVariable; +import com.tencent.supersonic.headless.api.pojo.enums.VariableValueType; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; +import org.stringtemplate.v4.ST; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.EMPTY; + +@Slf4j +public class SqlVariableParseUtils { + + public static final String REG_SENSITIVE_SQL = "drop\\s|alter\\s|grant\\s|insert\\s|replace\\s|delete\\s|" + + "truncate\\s|update\\s|remove\\s"; + public static final Pattern PATTERN_SENSITIVE_SQL = Pattern.compile(REG_SENSITIVE_SQL); + + public static final String APOSTROPHE = "'"; + + private static final char delimiter = '$'; + + public static String parse(String sql, List sqlVariables, List params) { + if (CollectionUtils.isEmpty(sqlVariables)) { + return sql; + } + Map queryParams = new HashMap<>(); + //1. handle default variable value + sqlVariables.forEach(variable -> { + queryParams.put(variable.getName().trim(), + getValues(variable.getValueType(), variable.getDefaultValues())); + }); + + //override by variable param + if (!CollectionUtils.isEmpty(params)) { + Map> map = + sqlVariables.stream().collect(Collectors.groupingBy(SqlVariable::getName)); + params.forEach(p -> { + if (map.containsKey(p.getName())) { + List list = map.get(p.getName()); + if (!CollectionUtils.isEmpty(list)) { + SqlVariable v = list.get(list.size() - 1); + queryParams.put(p.getName().trim(), getValue(v.getValueType(), p.getValue())); + } + } + }); + } + + queryParams.forEach((k, v) -> { + if (v instanceof List && ((List) v).size() > 0) { + v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString(); + } + queryParams.put(k, v); + }); + ST st = new ST(sql, delimiter, delimiter); + if (!CollectionUtils.isEmpty(queryParams)) { + queryParams.forEach(st::add); + } + return st.render(); + } + + public static List getValues(VariableValueType valueType, List values) { + if (CollectionUtils.isEmpty(values)) { + return new ArrayList<>(); + } + if (null != valueType) { + switch (valueType) { + case STRING: + return values.stream().map(String::valueOf) + .map(s -> s.startsWith(APOSTROPHE) && s.endsWith(APOSTROPHE) + ? s : String.join(EMPTY, APOSTROPHE, s, APOSTROPHE)) + .collect(Collectors.toList()); + case EXPR: + values.stream().map(String::valueOf).forEach(SqlVariableParseUtils::checkSensitiveSql); + return values.stream().map(String::valueOf).collect(Collectors.toList()); + case NUMBER: + return values.stream().map(String::valueOf).collect(Collectors.toList()); + default: + return values.stream().map(String::valueOf).collect(Collectors.toList()); + } + } + return values.stream().map(String::valueOf).collect(Collectors.toList()); + } + + public static Object getValue(VariableValueType valueType, String value) { + if (!StringUtils.isEmpty(value)) { + if (null != valueType) { + switch (valueType) { + case STRING: + return String.join(EMPTY, value.startsWith(APOSTROPHE) ? EMPTY : APOSTROPHE, + value, value.endsWith(APOSTROPHE) ? EMPTY : APOSTROPHE); + case NUMBER: + case EXPR: + default: + return value; + } + } + } + return value; + } + + public static void checkSensitiveSql(String sql) { + Matcher matcher = PATTERN_SENSITIVE_SQL.matcher(sql.toLowerCase()); + if (matcher.find()) { + String group = matcher.group(); + log.warn("Sensitive SQL operations are not allowed: {}", group.toUpperCase()); + throw new InvalidArgumentException("Sensitive SQL operations are not allowed: " + group.toUpperCase()); + } + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java index 7eeeb62ea..2f0b4f57b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java @@ -11,11 +11,13 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.server.service.MetricService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.Collection; @@ -28,6 +30,9 @@ import java.util.stream.Collectors; @Slf4j public class MetricDrillDownChecker { + @Autowired + private MetricService metricService; + @Around("execution(* com.tencent.supersonic.headless.core.parser.QueryParser.parse(..))") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { Object[] objects = joinPoint.getArgs(); @@ -52,7 +57,7 @@ public class MetricDrillDownChecker { List necessaryDimensions = getNecessaryDimensions(metric, semanticSchemaResp); List dimensionsMissing = getNecessaryDimensionMissing(necessaryDimensions, dimensionFields); if (!CollectionUtils.isEmpty(dimensionsMissing)) { - String errMsg = String.format("指标:%s 缺失必要维度:%s", metric.getName(), + String errMsg = String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(), dimensionsMissing.stream().map(DimensionResp::getName).collect(Collectors.toList())); throw new InvalidArgumentException(errMsg); } @@ -92,8 +97,9 @@ public class MetricDrillDownChecker { return true; } List relateDimensions = metricResps.stream() - .filter(metric -> !CollectionUtils.isEmpty(metric.getDrillDownDimensions())) - .map(metric -> metric.getDrillDownDimensions().stream() + .map(this::getDrillDownDimensions) + .filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions)) + .map(drillDownDimensions -> drillDownDimensions.stream() .map(DrillDownDimension::getDimensionId).collect(Collectors.toList())) .flatMap(Collection::stream) .map(id -> convertDimensionIdToBizName(id, semanticSchemaResp)) @@ -111,7 +117,7 @@ public class MetricDrillDownChecker { if (metric == null) { return Lists.newArrayList(); } - List drillDownDimensions = metric.getDrillDownDimensions(); + List drillDownDimensions = getDrillDownDimensions(metric); if (CollectionUtils.isEmpty(drillDownDimensions)) { return Lists.newArrayList(); } @@ -147,4 +153,8 @@ public class MetricDrillDownChecker { return dimension.getBizName(); } + private List getDrillDownDimensions(MetricResp metricResp) { + return metricService.getDrillDownDimension(metricResp.getId()); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index ee0bc723e..b73a758ac 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; @Slf4j public class ModelYamlManager { - public static DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) { + public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) { ModelDetail modelDetail = modelResp.getModelDetail(); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java index ef98e816f..37646f512 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java @@ -35,8 +35,6 @@ public class ViewDO { private String updatedBy; - private String filterSql; - private String queryConfig; private String admin; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java index a3ed936a1..03504b989 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; -import org.apache.ibatis.annotations.Mapper; - +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import java.util.List; +import org.apache.ibatis.annotations.Mapper; @Mapper public interface DimensionDOCustomMapper { @@ -16,4 +16,7 @@ public interface DimensionDOCustomMapper { void batchUpdateStatus(List dimensionDOS); List query(DimensionFilter dimensionFilter); + + List queryDimensions(DimensionsFilter dimensionsFilter); + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java index 0c785847e..1c93f17ba 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.pojo.MetricFilter; -import org.apache.ibatis.annotations.Mapper; - +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; +import org.apache.ibatis.annotations.Mapper; @Mapper public interface MetricDOCustomMapper { @@ -15,4 +15,6 @@ public interface MetricDOCustomMapper { List query(MetricFilter metricFilter); + List queryMetrics(MetricsFilter metricsFilter); + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java index df1cf295b..84ee389e5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java @@ -4,6 +4,7 @@ package com.tencent.supersonic.headless.server.persistence.repository; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import java.util.List; public interface DimensionRepository { @@ -19,4 +20,6 @@ public interface DimensionRepository { DimensionDO getDimensionById(Long id); List getDimension(DimensionFilter dimensionFilter); + + List getDimensions(DimensionsFilter dimensionsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java index 36fc060f3..4ba881c94 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; import com.tencent.supersonic.headless.server.pojo.MetricFilter; +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; public interface MetricRepository { @@ -21,6 +22,8 @@ public interface MetricRepository { List getMetric(MetricFilter metricFilter); + List getMetrics(MetricsFilter metricsFilter); + void saveDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO); void updateDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index f609abfeb..392c5547f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -5,9 +5,9 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCust import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; -import org.springframework.stereotype.Service; - +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import java.util.List; +import org.springframework.stereotype.Service; @Service public class DimensionRepositoryImpl implements DimensionRepository { @@ -52,4 +52,9 @@ public class DimensionRepositoryImpl implements DimensionRepository { return dimensionDOCustomMapper.query(dimensionFilter); } + @Override + public List getDimensions(DimensionsFilter dimensionsFilter) { + return dimensionDOCustomMapper.queryDimensions(dimensionsFilter); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java index cb8e5ae4d..3eabcf2b7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java @@ -8,9 +8,9 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper; import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefaultConfigDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.MetricFilter; -import org.springframework.stereotype.Component; - +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; +import org.springframework.stereotype.Component; @Component @@ -62,6 +62,11 @@ public class MetricRepositoryImpl implements MetricRepository { return metricDOCustomMapper.query(metricFilter); } + @Override + public List getMetrics(MetricsFilter metricsFilter) { + return metricDOCustomMapper.queryMetrics(metricsFilter); + } + @Override public void saveDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO) { metricQueryDefaultConfigDOMapper.insert(defaultConfigDO); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java new file mode 100644 index 000000000..e1e382967 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.headless.server.pojo; + +import java.util.List; +import lombok.Data; + +@Data +public class DimensionsFilter { + + private List modelIds; + + private List dimensionIds; + + private List dimensionNames; + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java new file mode 100644 index 000000000..256555a83 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.headless.server.pojo; + +import java.util.List; +import lombok.Data; + +@Data +public class MetricsFilter { + + private List modelIds; + + private List metricIds; + + private List metricNames; + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java new file mode 100644 index 000000000..b329d6054 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java @@ -0,0 +1,60 @@ +package com.tencent.supersonic.headless.server.pojo; + +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Data +public class ModelCluster { + + private static final String split = "_"; + + private Set modelIds = new LinkedHashSet<>(); + + private Set modelNames = new LinkedHashSet<>(); + + private String key; + + private String name; + + public static ModelCluster build(Set modelIds) { + ModelCluster modelCluster = new ModelCluster(); + modelCluster.setModelIds(modelIds); + modelCluster.setKey(StringUtils.join(modelIds, split)); + return modelCluster; + } + + public static ModelCluster build(String key) { + ModelCluster modelCluster = new ModelCluster(); + modelCluster.setModelIds(getModelIdFromKey(key)); + modelCluster.setKey(key); + return modelCluster; + } + + public void buildName(Map modelNameMap) { + modelNames = modelNameMap.entrySet().stream().filter(entry -> + modelIds.contains(entry.getKey())).map(Map.Entry::getValue) + .collect(Collectors.toSet()); + name = String.join(split, modelNames); + } + + public static Set getModelIdFromKey(String key) { + return Arrays.stream(key.split(split)) + .map(Long::parseLong).collect(Collectors.toSet()); + } + + public Long getFirstModel() { + if (CollectionUtils.isEmpty(modelIds)) { + return -1L; + } + return new ArrayList<>(modelIds).get(0); + } + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index 8a28c2c74..8e3211357 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -8,7 +8,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; import com.tencent.supersonic.headless.server.service.DatabaseService; -import java.util.Map; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -20,6 +19,7 @@ import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.util.List; +import java.util.Map; @RestController @RequestMapping("/api/semantic/database") @@ -49,8 +49,10 @@ public class DatabaseController { } @GetMapping("/{id}") - public DatabaseResp getDatabase(@PathVariable("id") Long id) { - return databaseService.getDatabase(id); + public DatabaseResp getDatabase(@PathVariable("id") Long id, HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return databaseService.getDatabase(id, user); } @GetMapping("/getDatabaseList") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index c08001777..7c222d4f1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -54,7 +54,7 @@ public class QueryController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq, true); + QuerySqlReq querySqlReq = queryStructReq.convert(true); return queryService.queryByReq(querySqlReq, user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index b255d50eb..99a1f3d97 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -15,6 +15,10 @@ public interface DatabaseService { SemanticQueryResp executeSql(String sql, Long id, User user); + DatabaseResp getDatabase(Long id, User user); + + DatabaseResp getDatabase(Long id); + Map> getDatabaseParameters(); boolean testConnect(DatabaseReq databaseReq, User user); @@ -25,8 +29,6 @@ public interface DatabaseService { void deleteDatabase(Long databaseId); - DatabaseResp getDatabase(Long id); - SemanticQueryResp getDbNames(Long id); SemanticQueryResp getTables(Long id, String db); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java index 2015def62..d5f88639a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java @@ -8,8 +8,8 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter; - import java.util.List; public interface DimensionService { @@ -30,6 +30,8 @@ public interface DimensionService { PageInfo queryDimension(PageDimensionReq pageDimensionReq); + List queryDimensions(DimensionsFilter dimensionsFilter); + void deleteDimension(Long id, User user); List getDimensionInModelCluster(Long modelId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java index 4d710bd4c..d974851da 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; - +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; import java.util.Set; @@ -50,4 +50,6 @@ public interface MetricService { MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user); void sendMetricEventBatch(List modelIds, EventType eventType); + + List queryMetrics(MetricsFilter metricsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index fe03efb2c..fe7d9d5d0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -37,6 +37,8 @@ public interface ModelService { List getModelByDomainIds(List domainIds); + List getAllModelByDomainIds(List domainIds); + ModelResp getModel(Long id); List getModelAdmin(Long id); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 86a51237f..2cb90fdac 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -15,6 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; @@ -26,6 +27,8 @@ public interface SchemaService { List fetchViewSchema(ViewFilterReq filter); + List fetchModelSchemaResps(List modelIds); + PageInfo queryDimension(PageDimensionReq pageDimensionReq, User user); PageInfo queryMetric(PageMetricReq pageMetricReq, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 910633085..1854434d8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; @@ -18,15 +19,16 @@ import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + @Slf4j @Service @@ -58,12 +60,12 @@ public class DatabaseServiceImpl implements DatabaseService { database.updatedBy(user.getName()); DatabaseConverter.convert(database, databaseDO); databaseRepository.updateDatabase(databaseDO); - return DatabaseConverter.convert(databaseDO); + return DatabaseConverter.convertWithPassword(databaseDO); } database.createdBy(user.getName()); databaseDO = DatabaseConverter.convert(database); databaseRepository.createDatabase(databaseDO); - return DatabaseConverter.convert(databaseDO); + return DatabaseConverter.convertWithPassword(databaseDO); } @Override @@ -108,7 +110,19 @@ public class DatabaseServiceImpl implements DatabaseService { @Override public DatabaseResp getDatabase(Long id) { DatabaseDO databaseDO = databaseRepository.getDatabase(id); - return DatabaseConverter.convert(databaseDO); + return DatabaseConverter.convertWithPassword(databaseDO); + } + + @Override + public DatabaseResp getDatabase(Long id, User user) { + DatabaseResp databaseResp = getDatabase(id); + if (!databaseResp.getAdmins().contains(user.getName()) + && !databaseResp.getViewers().contains(user.getName()) + && !databaseResp.getCreatedBy().equals(user.getName())) { + throw new InvalidPermissionException("您暂无查看该数据库详情的权限, 请联系创建人: " + + databaseResp.getCreatedBy()); + } + return databaseResp; } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 990757a1c..e3182e93f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -29,6 +29,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; @@ -37,19 +38,18 @@ import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service @Slf4j @@ -217,6 +217,12 @@ public class DimensionServiceImpl implements DimensionService { return dimensionRepository.getDimension(dimensionFilter); } + @Override + public List queryDimensions(DimensionsFilter dimensionsFilter) { + List dimensions = dimensionRepository.getDimensions(dimensionsFilter); + return convertList(dimensions, modelService.getModelMap()); + } + @Override public List getDimensions(MetaFilter metaFilter) { DimensionFilter dimensionFilter = new DimensionFilter(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index d0a62ed61..617de4893 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -66,7 +66,7 @@ public class DownloadServiceImpl implements DownloadService { private QueryService queryService; public DownloadServiceImpl(MetricService metricService, - DimensionService dimensionService, QueryService queryService) { + DimensionService dimensionService, QueryService queryService) { this.metricService = metricService; this.dimensionService = dimensionService; this.queryService = queryService; @@ -74,11 +74,11 @@ public class DownloadServiceImpl implements DownloadService { @Override public void downloadByStruct(DownloadStructReq downloadStructReq, - User user, HttpServletResponse response) throws Exception { + User user, HttpServletResponse response) throws Exception { String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); try { - QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq, true); + QuerySqlReq querySqlReq = downloadStructReq.convert(true); SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData()); @@ -92,7 +92,7 @@ public class DownloadServiceImpl implements DownloadService { @Override public void batchDownload(BatchDownloadReq batchDownloadReq, User user, - HttpServletResponse response) throws Exception { + HttpServletResponse response) throws Exception { String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); List metricIds = batchDownloadReq.getMetricIds(); @@ -109,8 +109,9 @@ public class DownloadServiceImpl implements DownloadService { metaFilter.setIds(metricIds); List metricResps = metricService.getMetrics(metaFilter); Map> metricMap = getMetricMap(metricResps); - List dimensionIds = metricResps.stream().map(MetricResp::getRelateDimension) - .map(RelateDimension::getDrillDownDimensions).flatMap(Collection::stream) + List dimensionIds = metricResps.stream() + .map(metricResp -> metricService.getDrillDownDimension(metricResp.getId())) + .flatMap(Collection::stream) .map(DrillDownDimension::getDimensionId).collect(Collectors.toList()); metaFilter.setIds(dimensionIds); Map dimensionRespMap = dimensionService.getDimensions(metaFilter) @@ -126,7 +127,7 @@ public class DownloadServiceImpl implements DownloadService { for (MetricResp metric : metrics) { try { DownloadStructReq downloadStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq); - QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq); + QuerySqlReq querySqlReq = downloadStructReq.convert(); querySqlReq.setNeedAuth(true); SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); @@ -192,7 +193,7 @@ public class DownloadServiceImpl implements DownloadService { } private List> buildData(List> headers, Map nameMap, - List> dataTransformed, String metricName) { + List> dataTransformed, String metricName) { List> data = Lists.newArrayList(); for (Map map : dataTransformed) { List row = Lists.newArrayList(); @@ -234,7 +235,7 @@ public class DownloadServiceImpl implements DownloadService { } private DownloadStructReq buildDownloadReq(List dimensionResps, MetricResp metricResp, - BatchDownloadReq batchDownloadReq) { + BatchDownloadReq batchDownloadReq) { DateConf dateConf = batchDownloadReq.getDateInfo(); Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); modelIds.add(metricResp.getModelId()); @@ -277,7 +278,7 @@ public class DownloadServiceImpl implements DownloadService { } private List getMetricRelaDimensions(MetricResp metricResp, - Map dimensionRespMap) { + Map dimensionRespMap) { if (metricResp.getRelateDimension() == null || CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { return Lists.newArrayList(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java index 5c517feb9..67b85cca6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java @@ -8,13 +8,14 @@ import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.server.service.KnowledgeService; import com.tencent.supersonic.headless.server.service.ViewService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -68,17 +69,19 @@ public class KnowledgeServiceImpl implements KnowledgeService { @Override public List getTerms(String text) { - return HanlpHelper.getTerms(text); + Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>()); + return HanlpHelper.getTerms(text, modelIdToViewIds); } @Override public List prefixSearch(String key, int limit, Set viewIds) { Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds)); - return prefixSearchByModel(key, limit, modelIdToViewIds.keySet()); + return prefixSearchByModel(key, limit, modelIdToViewIds); } - public List prefixSearchByModel(String key, int limit, Set models) { - return SearchService.prefixSearch(key, limit, models); + public List prefixSearchByModel(String key, int limit, + Map> modelIdToViewIds) { + return SearchService.prefixSearch(key, limit, modelIdToViewIds); } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index e026a6966..2896bbac4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -5,6 +5,7 @@ import com.alibaba.fastjson.TypeReference; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DataEvent; @@ -15,15 +16,15 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.ChatGptHelper; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; @@ -33,6 +34,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQuery import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetricFilter; +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import com.tencent.supersonic.headless.server.service.CollectService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; @@ -46,6 +48,7 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -184,9 +187,7 @@ public class MetricServiceImpl implements MetricService { public PageInfo queryMetric(PageMetricReq pageMetricReq, User user) { MetricFilter metricFilter = new MetricFilter(); BeanUtils.copyProperties(pageMetricReq, metricFilter); - Set domainResps = domainService.getDomainChildren(pageMetricReq.getDomainIds()); - List domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); - List modelResps = modelService.getModelByDomainIds(domainIds); + List modelResps = modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds()); List modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); pageMetricReq.getModelIds().addAll(modelIds); metricFilter.setModelIds(pageMetricReq.getModelIds()); @@ -230,28 +231,44 @@ public class MetricServiceImpl implements MetricService { } private List filterByField(List metricResps, List fields) { - List metricRespFiltered = Lists.newArrayList(); + Set metricRespFiltered = Sets.newHashSet(); for (MetricResp metricResp : metricResps) { - for (String field : fields) { - if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { - List ids = metricResp.getMetricDefineByMetricParams().getMetrics() - .stream().map(MetricParam::getId).collect(Collectors.toList()); - List metricById = metricResps.stream() - .filter(metric -> ids.contains(metric.getId())) - .collect(Collectors.toList()); - for (MetricResp metric : metricById) { - if (metric.getExpr().contains(field)) { - metricRespFiltered.add(metricResp); - } - } - } else { - if (metricResp.getExpr().contains(field)) { - metricRespFiltered.add(metricResp); - } + filterByField(metricResps, metricResp, fields, metricRespFiltered); + } + return new ArrayList<>(metricRespFiltered); + } + + private boolean filterByField(List metricResps, MetricResp metricResp, + List fields, Set metricRespFiltered) { + if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { + List ids = metricResp.getMetricDefineByMetricParams().getMetrics() + .stream().map(MetricParam::getId).collect(Collectors.toList()); + List metricById = metricResps.stream() + .filter(metric -> ids.contains(metric.getId())) + .collect(Collectors.toList()); + for (MetricResp metric : metricById) { + if (filterByField(metricResps, metric, fields, metricRespFiltered)) { + metricRespFiltered.add(metricResp); + return true; } } + } else if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) { + if (fields.stream().anyMatch(field -> metricResp.getExpr().contains(field))) { + metricRespFiltered.add(metricResp); + return true; + } + } else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) { + List measures = metricResp.getMetricDefineByMeasureParams().getMeasures(); + List fieldNameDepended = measures.stream().map(MeasureParam::getBizName) + //measure bizName = model bizName_fieldName + .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", "")) + .collect(Collectors.toList()); + if (fields.stream().anyMatch(fieldNameDepended::contains)) { + metricRespFiltered.add(metricResp); + return true; + } } - return metricRespFiltered; + return false; } @Override @@ -343,7 +360,12 @@ public class MetricServiceImpl implements MetricService { } if (metricResp.getRelateDimension() != null && !CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { - drillDownDimensions.addAll(metricResp.getRelateDimension().getDrillDownDimensions()); + for (DrillDownDimension drillDownDimension : metricResp.getRelateDimension().getDrillDownDimensions()) { + if (drillDownDimension.isInheritedFromModel() && !drillDownDimension.isNecessary()) { + continue; + } + drillDownDimensions.add(drillDownDimension); + } } ModelResp modelResp = modelService.getModel(metricResp.getModelId()); if (modelResp.getDrillDownDimensions() == null) { @@ -435,6 +457,12 @@ public class MetricServiceImpl implements MetricService { sendEventBatch(metricDOS, eventType); } + @Override + public List queryMetrics(MetricsFilter metricsFilter) { + List metricDOS = metricRepository.getMetrics(metricsFilter); + return convertList(metricDOS, new ArrayList<>()); + } + private void sendEventBatch(List metricDOS, EventType eventType) { List dataItems = metricDOS.stream().map(this::getDataItem) .collect(Collectors.toList()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index e9be8e689..f674656d7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -81,13 +81,13 @@ public class ModelServiceImpl implements ModelService { private DateInfoRepository dateInfoRepository; public ModelServiceImpl(ModelRepository modelRepository, - DatabaseService databaseService, - @Lazy DimensionService dimensionService, - @Lazy MetricService metricService, - DomainService domainService, - UserService userService, - ViewService viewService, - DateInfoRepository dateInfoRepository) { + DatabaseService databaseService, + @Lazy DimensionService dimensionService, + @Lazy MetricService metricService, + DomainService domainService, + UserService userService, + ViewService viewService, + DateInfoRepository dateInfoRepository) { this.modelRepository = modelRepository; this.databaseService = databaseService; this.dimensionService = dimensionService; @@ -350,6 +350,13 @@ public class ModelServiceImpl implements ModelService { domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList()); } + @Override + public List getAllModelByDomainIds(List domainIds) { + Set domainResps = domainService.getDomainChildren(domainIds); + List allDomainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); + return getModelByDomainIds(allDomainIds); + } + @Override public ModelResp getModel(Long id) { ModelDO modelDO = getModelDO(id); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index cd0385b62..ba8a3d3fd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Item; import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; @@ -44,13 +45,24 @@ import com.tencent.supersonic.headless.server.annotation.S2DataPermission; import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; +import com.tencent.supersonic.headless.server.pojo.ModelCluster; import com.tencent.supersonic.headless.server.service.AppService; import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.server.service.DimensionService; +import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -60,6 +72,7 @@ import javax.servlet.http.HttpServletRequest; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -79,6 +92,12 @@ public class QueryServiceImpl implements QueryService { private final QueryPlanner queryPlanner; + private final MetricService metricService; + + private final ModelService modelService; + + private final DimensionService dimensionService; + public QueryServiceImpl( StatUtils statUtils, QueryUtils queryUtils, @@ -88,7 +107,10 @@ public class QueryServiceImpl implements QueryService { QueryCache queryCache, SemanticSchemaManager semanticSchemaManager, DefaultQueryParser queryParser, - QueryPlanner queryPlanner) { + QueryPlanner queryPlanner, + MetricService metricService, + ModelService modelService, + DimensionService dimensionService) { this.statUtils = statUtils; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; @@ -98,6 +120,9 @@ public class QueryServiceImpl implements QueryService { this.semanticSchemaManager = semanticSchemaManager; this.queryParser = queryParser; this.queryPlanner = queryPlanner; + this.metricService = metricService; + this.modelService = modelService; + this.dimensionService = dimensionService; } @Override @@ -241,8 +266,130 @@ public class QueryServiceImpl implements QueryService { } @Override - public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception { - return null; + public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) { + QueryStructReq queryStructReq = buildQueryStructReq(queryMetricReq); + return queryByReq(queryStructReq.convert(), user); + } + + private QueryStructReq buildQueryStructReq(QueryMetricReq queryMetricReq) { + //1. If a domainId exists, the modelIds obtained from the domainId. + Set modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq); + + //2. get metrics and dimensions + List metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId); + + List dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId); + + //3. choose ModelCluster + Set modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps); + ModelCluster modelCluster = getModelCluster(metricResps, modelIds); + + //4. set groups + List dimensionBizNames = dimensionResps.stream() + .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) + .map(entry -> entry.getBizName()).collect(Collectors.toList()); + + QueryStructReq queryStructReq = new QueryStructReq(); + if (CollectionUtils.isNotEmpty(dimensionBizNames)) { + queryStructReq.setGroups(dimensionBizNames); + } + //5. set aggregators + List metricBizNames = metricResps.stream() + .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) + .map(entry -> entry.getBizName()).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(metricBizNames)) { + throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics"); + } + List aggregators = new ArrayList<>(); + for (String metricBizName : metricBizNames) { + Aggregator aggregator = new Aggregator(); + aggregator.setColumn(metricBizName); + aggregators.add(aggregator); + } + queryStructReq.setAggregators(aggregators); + queryStructReq.setLimit(queryMetricReq.getLimit()); + //6. set modelIds + for (Long modelId : modelCluster.getModelIds()) { + queryStructReq.addModelId(modelId); + } + //7. set dateInfo + queryStructReq.setDateInfo(queryMetricReq.getDateInfo()); + return queryStructReq; + } + + private QueryStructReq buildQueryStructReq(List dimensionResps, + MetricResp metricResp, DateConf dateConf, Long limit) { + Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); + modelIds.add(metricResp.getModelId()); + QueryStructReq queryStructReq = new QueryStructReq(); + queryStructReq.setGroups(dimensionResps.stream() + .map(DimensionResp::getBizName).collect(Collectors.toList())); + queryStructReq.getGroups().add(0, getTimeDimension(dateConf)); + Aggregator aggregator = new Aggregator(); + aggregator.setColumn(metricResp.getBizName()); + queryStructReq.setAggregators(Lists.newArrayList(aggregator)); + queryStructReq.setDateInfo(dateConf); + queryStructReq.setModelIds(modelIds); + queryStructReq.setLimit(limit); + return queryStructReq; + } + + private ModelCluster getModelCluster(List metricResps, Set modelIds) { + Map modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds)); + + Map> modelClusterToMatchCount = new HashMap<>(); + for (ModelCluster modelCluster : modelClusterMap.values()) { + for (MetricResp metricResp : metricResps) { + if (modelCluster.getModelIds().contains(metricResp.getModelId())) { + modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>()) + .add(metricResp); + } + } + } + String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream() + .max(Comparator.comparingInt(entry -> entry.getValue().size())) + .map(Map.Entry::getKey) + .orElse(null); + + return modelClusterMap.get(keyWithMaxSize); + } + + private Set getModelIds(Set modelIdsByDomainId, List metricResps, + List dimensionResps) { + Set result = new HashSet<>(); + if (CollectionUtils.isNotEmpty(modelIdsByDomainId)) { + result.addAll(modelIdsByDomainId); + return result; + } + Set metricModelIds = metricResps.stream().map(entry -> entry.getModelId()) + .collect(Collectors.toSet()); + result.addAll(metricModelIds); + + Set dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId()) + .collect(Collectors.toSet()); + result.addAll(dimensionModelIds); + return result; + } + + private List getDimensionResps(QueryMetricReq queryMetricReq, Set modelIds) { + DimensionsFilter dimensionsFilter = new DimensionsFilter(); + BeanUtils.copyProperties(queryMetricReq, dimensionsFilter); + dimensionsFilter.setModelIds(new ArrayList<>(modelIds)); + List dimensionResps = dimensionService.queryDimensions(dimensionsFilter); + return dimensionResps; + } + + private List getMetricResps(QueryMetricReq queryMetricReq, Set modelIds) { + MetricsFilter metricsFilter = new MetricsFilter(); + BeanUtils.copyProperties(queryMetricReq, metricsFilter); + metricsFilter.setModelIds(new ArrayList<>(modelIds)); + return metricService.queryMetrics(metricsFilter); + } + + private Set getModelIdsByDomainId(QueryMetricReq queryMetricReq) { + List modelResps = modelService.getAllModelByDomainIds( + Collections.singletonList(queryMetricReq.getDomainId())); + return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet()); } private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception { @@ -271,23 +418,6 @@ public class QueryServiceImpl implements QueryService { return appService.getApp(appId); } - private QueryStructReq buildQueryStructReq(List dimensionResps, - MetricResp metricResp, DateConf dateConf, Long limit) { - Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); - modelIds.add(metricResp.getModelId()); - QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setGroups(dimensionResps.stream() - .map(DimensionResp::getBizName).collect(Collectors.toList())); - queryStructReq.getGroups().add(0, getTimeDimension(dateConf)); - Aggregator aggregator = new Aggregator(); - aggregator.setColumn(metricResp.getBizName()); - queryStructReq.setAggregators(Lists.newArrayList(aggregator)); - queryStructReq.setDateInfo(dateConf); - queryStructReq.setModelIds(modelIds); - queryStructReq.setLimit(limit); - return queryStructReq; - } - private String getTimeDimension(DateConf dateConf) { if (Constants.MONTH.equals(dateConf.getPeriod())) { return TimeDimensionEnum.MONTH.getName(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index c92a09447..a1cf5160f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -64,6 +64,13 @@ public class SchemaServiceImpl implements SchemaService { protected final Cache> itemUseCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); + + protected final Cache> viewSchemaCache = + CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build(); + + protected final Cache semanticSchemaCache = + CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build(); + private final StatUtils statUtils; private final ModelService modelService; private final DimensionService dimensionService; @@ -91,6 +98,22 @@ public class SchemaServiceImpl implements SchemaService { @SneakyThrows @Override public List fetchViewSchema(ViewFilterReq filter) { + List viewList = viewSchemaCache.getIfPresent(filter); + if (CollectionUtils.isEmpty(viewList)) { + viewList = buildViewSchema(filter); + viewSchemaCache.put(filter, viewList); + } + return viewList; + } + + public ViewSchemaResp fetchViewSchema(Long viewId) { + if (viewId == null) { + return null; + } + return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null); + } + + public List buildViewSchema(ViewFilterReq filter) { List viewSchemaResps = new ArrayList<>(); List viewIds = filter.getViewIds(); MetaFilter metaFilter = new MetaFilter(); @@ -127,13 +150,6 @@ public class SchemaServiceImpl implements SchemaService { return viewSchemaResps; } - public ViewSchemaResp fetchViewSchema(Long viewId) { - if (viewId == null) { - return null; - } - return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null); - } - public List fetchModelSchemaResps(List modelIds) { List modelSchemaResps = Lists.newArrayList(); if (CollectionUtils.isEmpty(modelIds)) { @@ -258,8 +274,7 @@ public class SchemaServiceImpl implements SchemaService { return viewService.getViewList(metaFilter); } - @Override - public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) { + public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) { SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp(); semanticSchemaResp.setViewId(schemaFilterReq.getViewId()); semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds()); @@ -294,6 +309,16 @@ public class SchemaServiceImpl implements SchemaService { return semanticSchemaResp; } + @Override + public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) { + SemanticSchemaResp semanticSchemaResp = semanticSchemaCache.getIfPresent(schemaFilterReq); + if (semanticSchemaResp == null) { + semanticSchemaResp = buildSemanticSchema(schemaFilterReq); + semanticSchemaCache.put(schemaFilterReq, semanticSchemaResp); + } + return semanticSchemaResp; + } + @SneakyThrows @Override public List getStatInfo(ItemUseReq itemUseReq) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java index 9b60a20a9..b11eb12e2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java @@ -10,17 +10,29 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.ViewDetail; import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO; import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; +import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ViewService; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.Arrays; import java.util.Comparator; import java.util.Date; @@ -29,12 +41,8 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service public class ViewServiceImpl @@ -46,21 +54,33 @@ public class ViewServiceImpl @Autowired private DomainService domainService; + @Lazy + @Autowired + private DimensionService dimensionService; + + @Lazy + @Autowired + private MetricService metricService; + @Override public ViewResp save(ViewReq viewReq, User user) { viewReq.createdBy(user.getName()); ViewDO viewDO = convert(viewReq); viewDO.setStatus(StatusEnum.ONLINE.getCode()); + ViewResp viewResp = convert(viewDO); + conflictCheck(viewResp); save(viewDO); - return convert(viewDO); + return viewResp; } @Override public ViewResp update(ViewReq viewReq, User user) { viewReq.updatedBy(user.getName()); ViewDO viewDO = convert(viewReq); + ViewResp viewResp = convert(viewDO); + conflictCheck(viewResp); updateById(viewDO); - return convert(viewDO); + return viewResp; } @Override @@ -78,6 +98,9 @@ public class ViewServiceImpl if (!CollectionUtils.isEmpty(metaFilter.getIds())) { wrapper.lambda().in(ViewDO::getId, metaFilter.getIds()); } + if (metaFilter.getStatus() != null) { + wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus()); + } wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode()); return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); } @@ -175,4 +198,46 @@ public class ViewServiceImpl viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId()))) .collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList()))); } + + private void conflictCheck(ViewResp viewResp) { + List allDimensionIds = viewResp.getAllDimensions(); + List allMetricIds = viewResp.getAllMetrics(); + MetaFilter metaFilter = new MetaFilter(); + if (!CollectionUtils.isEmpty(allDimensionIds)) { + metaFilter.setIds(allDimensionIds); + List dimensionResps = dimensionService.getDimensions(metaFilter); + List duplicateDimensionNames = findDuplicates(dimensionResps, DimensionResp::getName); + List duplicateDimensionBizNames = findDuplicates(dimensionResps, DimensionResp::getBizName); + if (!duplicateDimensionNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的维度名: " + duplicateDimensionNames); + } + if (!duplicateDimensionBizNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的维度英文名: " + duplicateDimensionBizNames); + } + } + if (!CollectionUtils.isEmpty(allMetricIds)) { + metaFilter.setIds(allMetricIds); + List metricResps = metricService.getMetrics(metaFilter); + List duplicateMetricNames = findDuplicates(metricResps, MetricResp::getName); + List duplicateMetricBizNames = findDuplicates(metricResps, MetricResp::getBizName); + + if (!duplicateMetricNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的指标名: " + duplicateMetricNames); + } + if (!duplicateMetricBizNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的指标英文名: " + duplicateMetricBizNames); + } + } + } + + private List findDuplicates(List list, Function keyExtractor) { + return list.stream() + .collect(Collectors.groupingBy(keyExtractor, Collectors.counting())) + .entrySet().stream() + .filter(entry -> entry.getValue() > 1) + .map(Map.Entry::getKey) + .map(Object::toString) + .collect(Collectors.toList()); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java index 047e30c5d..064880845 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java @@ -63,7 +63,6 @@ public class DatabaseConverter { BeanUtils.copyProperties(databaseDO, databaseResp); ConnectInfo connectInfo = JSONObject.parseObject(databaseDO.getConfig(), ConnectInfo.class); databaseResp.setUrl(connectInfo.getUrl()); - databaseResp.setPassword(connectInfo.getPassword()); databaseResp.setUsername(connectInfo.getUserName()); databaseResp.setDatabase(connectInfo.getDatabase()); if (StringUtils.isNotBlank(databaseDO.getAdmin())) { @@ -75,4 +74,11 @@ public class DatabaseConverter { return databaseResp; } + public static DatabaseResp convertWithPassword(DatabaseDO databaseDO) { + DatabaseResp databaseResp = convert(databaseDO); + ConnectInfo connectInfo = JSONObject.parseObject(databaseDO.getConfig(), ConnectInfo.class); + databaseResp.setPassword(connectInfo.getPassword()); + return databaseResp; + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 4de9272a4..8c69a0cf4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -1,5 +1,10 @@ package com.tencent.supersonic.headless.server.utils; +import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; +import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.SPACE; + import com.google.common.base.Strings; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; @@ -31,11 +36,6 @@ import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.QueryService; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -48,11 +48,10 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.StringJoiner; - -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Component public class DictUtils { @@ -79,9 +78,9 @@ public class DictUtils { private final ModelService modelService; public DictUtils(DimensionService dimensionService, - MetricService metricService, - QueryService queryService, - ModelService modelService) { + MetricService metricService, + QueryService queryService, + ModelService modelService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; @@ -222,7 +221,7 @@ public class DictUtils { && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { // 查询默认指标 QueryStructReq queryStructReq = generateQueryStruct(dictItemResp); - return queryStructReq.convert(queryStructReq, true); + return queryStructReq.convert(true); } // count(1) 作为指标 return constructQuerySqlReq(dictItemResp); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index 282b6ff39..98cbbea9e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -72,6 +72,7 @@ public class MetricConverter { ModelResp modelResp = modelMap.get(metricDO.getModelId()); if (modelResp != null) { metricResp.setModelName(modelResp.getName()); + metricResp.setModelBizName(modelResp.getBizName()); metricResp.setDomainId(modelResp.getDomainId()); } metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId())); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java new file mode 100644 index 000000000..161d34889 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java @@ -0,0 +1,47 @@ +package com.tencent.supersonic.headless.server.utils; + + +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.server.pojo.ModelCluster; +import com.tencent.supersonic.headless.server.service.SchemaService; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class ModelClusterBuilder { + + public static Map buildModelClusters(List modelIds) { + SchemaService schemaService = ContextUtils.getBean(SchemaService.class); + List modelSchemaResps = schemaService.fetchModelSchemaResps(modelIds); + Map modelIdToModelSchema = modelSchemaResps.stream() + .collect(Collectors.toMap(ModelSchemaResp::getId, value -> value, (k1, k2) -> k1)); + + Set visited = new HashSet<>(); + List> modelClusters = new ArrayList<>(); + for (ModelSchemaResp model : modelSchemaResps) { + if (!visited.contains(model.getId())) { + Set modelCluster = new HashSet<>(); + dfs(model, modelIdToModelSchema, visited, modelCluster); + modelClusters.add(modelCluster); + } + } + return modelClusters.stream().map(ModelCluster::build) + .collect(Collectors.toMap(ModelCluster::getKey, value -> value, (k1, k2) -> k1)); + } + + private static void dfs(ModelSchemaResp model, Map modelMap, + Set visited, Set modelCluster) { + visited.add(model.getId()); + modelCluster.add(model.getId()); + for (Long neighborId : model.getModelClusterSet()) { + if (!visited.contains(neighborId)) { + dfs(modelMap.get(neighborId), modelMap, visited, modelCluster); + } + } + } + +} \ No newline at end of file diff --git a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index d9fda8391..4f82ebd56 100644 --- a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -167,4 +167,40 @@ + + + diff --git a/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml index 3a8fca6b6..ba3eede7d 100644 --- a/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml @@ -2,27 +2,29 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - + + @@ -56,14 +58,16 @@ - id, model_id, name, biz_name, description, status, sensitive_level, type, created_at, + id + , model_id, name, biz_name, description, status, sensitive_level, type, created_at, created_by, updated_at, updated_by, data_format_type, data_format, alias, tags, define_type type_params - + insert into s2_metric (model_id, name, biz_name, description, type,status,sensitive_level, created_at, created_by, updated_at, @@ -94,9 +98,9 @@ + + + diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java new file mode 100644 index 000000000..c9e119082 --- /dev/null +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/SqlVariableParseUtilsTest.java @@ -0,0 +1,71 @@ +package com.tencent.supersonic.headless.server.utils; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.Param; +import com.tencent.supersonic.headless.api.pojo.SqlVariable; +import com.tencent.supersonic.headless.api.pojo.enums.VariableValueType; +import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import java.util.List; + +public class SqlVariableParseUtilsTest { + + @Test + void testParseSql_defaultVariableValue() { + String sql = "select * from t_$interval$ where id = $id$ and name = $name$"; + List variables = Lists.newArrayList(mockNumSqlVariable(), + mockExprSqlVariable(), mockStrSqlVariable()); + String actualSql = SqlVariableParseUtils.parse(sql, variables, Lists.newArrayList()); + String expectedSql = "select * from t_d where id = 1 and name = 'tom'"; + Assertions.assertEquals(expectedSql, actualSql); + } + + @Test + void testParseSql() { + String sql = "select * from t_$interval$ where id = $id$ and name = $name$"; + List variables = Lists.newArrayList(mockNumSqlVariable(), + mockExprSqlVariable(), mockStrSqlVariable()); + List params = Lists.newArrayList(mockIdParam(), mockNameParam(), mockIntervalParam()); + String actualSql = SqlVariableParseUtils.parse(sql, variables, params); + String expectedSql = "select * from t_wk where id = 2 and name = 'alice'"; + Assertions.assertEquals(expectedSql, actualSql); + } + + private SqlVariable mockNumSqlVariable() { + return mockSqlVariable("id", VariableValueType.NUMBER, 1); + } + + private SqlVariable mockStrSqlVariable() { + return mockSqlVariable("name", VariableValueType.STRING, "tom"); + } + + private SqlVariable mockExprSqlVariable() { + return mockSqlVariable("interval", VariableValueType.EXPR, "d"); + } + + private SqlVariable mockSqlVariable(String name, VariableValueType variableValueType, Object value) { + SqlVariable sqlVariable = new SqlVariable(); + sqlVariable.setName(name); + sqlVariable.setValueType(variableValueType); + sqlVariable.setDefaultValues(Lists.newArrayList(value)); + return sqlVariable; + } + + private Param mockIdParam() { + return mockParam("id", "2"); + } + + private Param mockNameParam() { + return mockParam("name", "alice"); + } + + private Param mockIntervalParam() { + return mockParam("interval", "wk"); + } + + private Param mockParam(String name, String value) { + return new Param(name, value); + } + +} diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java index c54d033a4..76f38b21d 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java @@ -224,7 +224,7 @@ public class BenchMarkDemoDataLoader { new ViewModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()), new ViewModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()), new ViewModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()), - new ViewModelConfig(8L, Lists.newArrayList(13L, 14L, 15L), Lists.newArrayList(8L, 9L)) + new ViewModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L)) ); ViewDetail viewDetail = new ViewDetail(); viewDetail.setViewModelConfigs(viewModelConfigs); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index 240dc7e28..d4861b53d 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -23,7 +23,6 @@ import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.server.service.KnowledgeService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -53,8 +52,6 @@ public class ChatDemoLoader implements CommandLineRunner { private AgentService agentService; @Autowired private SysParameterService sysParameterService; - @Autowired - private KnowledgeService knowledgeService; @Value("${demo.enabled:false}") private boolean demoEnabled; diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java b/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java index 46cb9b42e..efdd52353 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java @@ -1,10 +1,10 @@ package com.tencent.supersonic; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; import com.tencent.supersonic.chat.core.parser.JavaLLMProxy; import com.tencent.supersonic.chat.core.parser.sql.llm.SqlExamplarLoader; import com.tencent.supersonic.chat.core.parser.sql.llm.SqlExample; import com.tencent.supersonic.chat.core.utils.ComponentFactory; +import com.tencent.supersonic.common.config.EmbeddingConfig; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -20,7 +20,7 @@ public class EmbeddingInitListener implements CommandLineRunner { @Autowired private SqlExamplarLoader sqlExamplarLoader; @Autowired - private OptimizationConfig optimizationConfig; + private EmbeddingConfig embeddingConfig; @Override public void run(String... args) { @@ -31,7 +31,7 @@ public class EmbeddingInitListener implements CommandLineRunner { try { if (ComponentFactory.getLLMProxy() instanceof JavaLLMProxy) { List sqlExamples = sqlExamplarLoader.getSqlExamples(); - String collectionName = optimizationConfig.getText2sqlCollectionName(); + String collectionName = embeddingConfig.getText2sqlCollectionName(); sqlExamplarLoader.addEmbeddingStore(sqlExamples, collectionName); } } catch (Exception e) { diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index a8fbd7053..8cf1b54fd 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -46,4 +46,4 @@ com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\ com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ - com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore \ No newline at end of file + com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore diff --git a/launchers/standalone/src/main/resources/application-local.yaml b/launchers/standalone/src/main/resources/application-local.yaml index b4bf0c8d3..1acba84ae 100644 --- a/launchers/standalone/src/main/resources/application-local.yaml +++ b/launchers/standalone/src/main/resources/application-local.yaml @@ -42,6 +42,9 @@ metric: mybatis: mapper-locations=classpath:mappers/custom/*.xml,classpath*:/mappers/*.xml +corrector: + additional: + information: true llm: parser: diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index c4bae89fb..3e5c8a916 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -183,7 +183,10 @@ CREATE TABLE s2_view( created_at datetime, created_by VARCHAR(255), updated_at datetime, - updated_by VARCHAR(255) + updated_by VARCHAR(255), + query_config VARCHAR(3000), + `admin` varchar(3000) DEFAULT NULL, + `admin_org` varchar(3000) DEFAULT NULL )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; alter table s2_plugin change column model `view` varchar(100); diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt index 9871036d4..b01cf97ae 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _2_7 100 -陈奕迅 _2_7 100 -林俊杰 _2_7 100 -张碧晨 _2_7 100 -程响 _2_7 100 -Taylor#Swift _2_7 100 -内地 _2_4 100 -欧美 _2_4 100 -港台 _2_4 100 -流行 _2_6 100 -国风 _2_6 100 \ No newline at end of file +周杰伦 _4_7 100 +陈奕迅 _4_7 100 +林俊杰 _4_7 100 +张碧晨 _4_7 100 +程响 _4_7 100 +Taylor#Swift _4_7 100 +内地 _4_4 100 +欧美 _4_4 100 +港台 _4_4 100 +流行 _4_6 100 +国风 _4_6 100 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt index d0d411751..e442a219c 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _2_3 52 -p2 _2_3 47 -p3 _2_3 31 -p4 _2_3 36 -p5 _2_3 50 -p6 _2_3 38 \ No newline at end of file +p1 _3_3 52 +p2 _3_3 47 +p3 _3_3 31 +p4 _3_3 36 +p5 _3_3 50 +p6 _3_3 38 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt index 6699a0fc8..3bc23d613 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt @@ -1,9 +1,9 @@ -周杰伦 _2_7 9000 -周深 _2_7 8000 -周传雄 _2_7 7000 -周华建 _2_7 6000 -陈奕迅 _2_7 8000 -林俊杰 _2_7 7000 -张碧晨 _2_7 7000 -程响 _2_7 7000 -Taylor#Swift _2_7 7000 \ No newline at end of file +周杰伦 _4_7 9000 +周深 _4_7 8000 +周传雄 _4_7 7000 +周华建 _4_7 6000 +陈奕迅 _4_7 8000 +林俊杰 _4_7 7000 +张碧晨 _4_7 7000 +程响 _4_7 7000 +Taylor#Swift _4_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt index cce180c73..605735e47 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt @@ -1,4 +1,4 @@ -美国 _3_8 1 -加拿大 _3_8 1 -锡尔赫特、吉大港、库斯蒂亚 _3_8 1 -孟加拉国 _3_8 3 +美国 _5_8 1 +加拿大 _5_8 1 +锡尔赫特、吉大港、库斯蒂亚 _5_8 1 +孟加拉国 _5_8 3 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt index bd4498b5d..ad0bf23b0 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt @@ -1,6 +1,6 @@ -现代 _3_9 1 -tagore _3_9 1 -蓝调 _3_9 1 -流行 _3_9 1 -民间 _3_9 1 -nazrul _3_9 1 +现代 _5_9 1 +tagore _5_9 1 +蓝调 _5_9 1 +流行 _5_9 1 +民间 _5_9 1 +nazrul _5_9 1 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt index 9e65454db..fbdf86ead 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt @@ -1,4 +1,4 @@ -美国 _3_11 1 -印度 _3_11 2 -英国 _3_11 1 -孟加拉国 _3_11 2 +美国 _6_11 1 +印度 _6_11 2 +英国 _6_11 1 +孟加拉国 _6_11 2 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt index 1f6344ad7..51a3b1edc 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt @@ -1,2 +1,2 @@ -男性 _3_12 3 -女性 _3_12 3 +男性 _6_12 3 +女性 _6_12 3 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt index 0ddbbfdc3..71561720f 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt @@ -1,2 +1,2 @@ -mp4 _3_14 4 -mp3 _3_14 2 +mp4 _7_14 4 +mp3 _7_14 2 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt index 65cb9fbf7..f7dd4e308 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt @@ -1,4 +1,4 @@ -美国 _3_17 1 -印度 _3_17 2 -英国 _3_17 1 -孟加拉国 _3_17 2 +美国 _8_17 1 +印度 _8_17 2 +英国 _8_17 1 +孟加拉国 _8_17 2 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt index 5705011ee..360d9b05d 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt @@ -1,2 +1,2 @@ -英文 _3_18 2 -孟加拉语 _3_18 4 +英文 _8_18 2 +孟加拉语 _8_18 4 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt index 1235c3ad6..d96cbf33f 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt @@ -1,6 +1,6 @@ -阿米·奥帕尔·霍伊 _3_16 1 -我的爱 _3_16 1 -打败它 _3_16 1 -阿杰伊阿卡什 _3_16 1 -Tumi#长袍#尼罗布 _3_16 1 -舒克诺#帕塔尔#努普尔#帕埃 _3_16 1 +阿米·奥帕尔·霍伊 _8_16 1 +我的爱 _8_16 1 +打败它 _8_16 1 +阿杰伊阿卡什 _8_16 1 +Tumi#长袍#尼罗布 _8_16 1 +舒克诺#帕塔尔#努普尔#帕埃 _8_16 1 diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 475ccab24..21a4154f0 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -569,7 +569,6 @@ CREATE TABLE IF NOT EXISTS `s2_view` ( created_by VARCHAR(255), updated_at TIMESTAMP, updated_by VARCHAR(255), - filter_sql VARCHAR(1000), query_config VARCHAR(3000), `admin` varchar(3000) DEFAULT NULL, `admin_org` varchar(3000) DEFAULT NULL diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index acc6b5b89..a8e38fad5 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -494,7 +494,6 @@ CREATE TABLE s2_view created_by VARCHAR(255), updated_at datetime, updated_by VARCHAR(255), - filter_sql VARCHAR(1000), query_config VARCHAR(3000), `admin` varchar(3000) DEFAULT NULL, `admin_org` varchar(3000) DEFAULT NULL diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java new file mode 100644 index 000000000..02e6d90a8 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java @@ -0,0 +1,33 @@ +package com.tencent.supersonic.headless; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; +import com.tencent.supersonic.headless.server.service.ModelService; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +public class ModelSchemaTest extends BaseTest { + + @Autowired + private ModelService modelService; + + @Test + void testGetUnAvailableItem() { + FieldRemovedReq fieldRemovedReq = new FieldRemovedReq(); + fieldRemovedReq.setModelId(2L); + fieldRemovedReq.setFields(Lists.newArrayList("pv")); + UnAvailableItemResp unAvailableItemResp = modelService.getUnAvailableItem(fieldRemovedReq); + List expectedUnAvailableMetricId = Lists.newArrayList(1L, 3L); + List actualUnAvailableMetricId = unAvailableItemResp.getMetricResps() + .stream().map(MetricResp::getId).sorted(Comparator.naturalOrder()).collect(Collectors.toList()); + Assertions.assertEquals(expectedUnAvailableMetricId, actualUnAvailableMetricId); + } + +} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java new file mode 100644 index 000000000..f8aac7726 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java @@ -0,0 +1,62 @@ +package com.tencent.supersonic.headless; + +import static org.junit.Assert.assertThrows; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import java.util.Arrays; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +public class QueryByMetricTest extends BaseTest { + + @Test + public void testWithMetricAndDimensionBizNames() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); + queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + } + + @Test + public void testWithMetricAndDimensionNames() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setMetricNames(Arrays.asList("停留时长", "访问次数")); + queryMetricReq.setDimensionNames(Arrays.asList("用户", "部门")); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + } + + @Test + public void testWithDomainId() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setDomainId(1L); + queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); + queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + + queryMetricReq.setDomainId(2L); + queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); + queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); + assertThrows(IllegalArgumentException.class, + () -> queryService.queryByMetric(queryMetricReq, User.getFakeUser())); + } + + @Test + public void testWithMetricAndDimensionIds() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setDomainId(1L); + queryMetricReq.setMetricIds(Arrays.asList(1L, 4L)); + queryMetricReq.setDimensionIds(Arrays.asList(1L, 2L)); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + } + +} diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt index 9871036d4..b01cf97ae 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _2_7 100 -陈奕迅 _2_7 100 -林俊杰 _2_7 100 -张碧晨 _2_7 100 -程响 _2_7 100 -Taylor#Swift _2_7 100 -内地 _2_4 100 -欧美 _2_4 100 -港台 _2_4 100 -流行 _2_6 100 -国风 _2_6 100 \ No newline at end of file +周杰伦 _4_7 100 +陈奕迅 _4_7 100 +林俊杰 _4_7 100 +张碧晨 _4_7 100 +程响 _4_7 100 +Taylor#Swift _4_7 100 +内地 _4_4 100 +欧美 _4_4 100 +港台 _4_4 100 +流行 _4_6 100 +国风 _4_6 100 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt index d0d411751..e442a219c 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _2_3 52 -p2 _2_3 47 -p3 _2_3 31 -p4 _2_3 36 -p5 _2_3 50 -p6 _2_3 38 \ No newline at end of file +p1 _3_3 52 +p2 _3_3 47 +p3 _3_3 31 +p4 _3_3 36 +p5 _3_3 50 +p6 _3_3 38 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt index 6699a0fc8..3bc23d613 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt @@ -1,9 +1,9 @@ -周杰伦 _2_7 9000 -周深 _2_7 8000 -周传雄 _2_7 7000 -周华建 _2_7 6000 -陈奕迅 _2_7 8000 -林俊杰 _2_7 7000 -张碧晨 _2_7 7000 -程响 _2_7 7000 -Taylor#Swift _2_7 7000 \ No newline at end of file +周杰伦 _4_7 9000 +周深 _4_7 8000 +周传雄 _4_7 7000 +周华建 _4_7 6000 +陈奕迅 _4_7 8000 +林俊杰 _4_7 7000 +张碧晨 _4_7 7000 +程响 _4_7 7000 +Taylor#Swift _4_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt index 6402c14f5..605735e47 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt @@ -1,4 +1,4 @@ 美国 _5_8 1 加拿大 _5_8 1 锡尔赫特、吉大港、库斯蒂亚 _5_8 1 -孟加拉国 _5_8 3 \ No newline at end of file +孟加拉国 _5_8 3 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt index 6cef5f046..ad0bf23b0 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt @@ -3,4 +3,4 @@ tagore _5_9 1 蓝调 _5_9 1 流行 _5_9 1 民间 _5_9 1 -nazrul _5_9 1 \ No newline at end of file +nazrul _5_9 1 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt index b5458d1ab..fbdf86ead 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt @@ -1,4 +1,4 @@ -美国 _6_10 1 -印度 _6_10 2 -英国 _6_10 1 -孟加拉国 _6_10 2 \ No newline at end of file +美国 _6_11 1 +印度 _6_11 2 +英国 _6_11 1 +孟加拉国 _6_11 2 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt index 33944bd5d..51a3b1edc 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt @@ -1,2 +1,2 @@ -男性 _6_11 3 -女性 _6_11 3 \ No newline at end of file +男性 _6_12 3 +女性 _6_12 3 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt index 46f76c558..71561720f 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt @@ -1,2 +1,2 @@ mp4 _7_14 4 -mp3 _7_14 2 \ No newline at end of file +mp3 _7_14 2 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt index 78ea079a1..f7dd4e308 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt @@ -1,4 +1,4 @@ -美国 _8_16 1 -印度 _8_16 2 -英国 _8_16 1 -孟加拉国 _8_16 2 \ No newline at end of file +美国 _8_17 1 +印度 _8_17 2 +英国 _8_17 1 +孟加拉国 _8_17 2 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt index 86a5882f3..360d9b05d 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt @@ -1,2 +1,2 @@ -英文 _8_17 2 -孟加拉语 _8_17 4 \ No newline at end of file +英文 _8_18 2 +孟加拉语 _8_18 4 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt index 0c00c7fa7..d96cbf33f 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt @@ -1,6 +1,6 @@ -阿米·奥帕尔·霍伊 _8_19 1 -我的爱 _8_19 1 -打败它 _8_19 1 -阿杰伊阿卡什 _8_19 1 -Tumi#长袍#尼罗布 _8_19 1 -舒克诺#帕塔尔#努普尔#帕埃 _8_19 1 \ No newline at end of file +阿米·奥帕尔·霍伊 _8_16 1 +我的爱 _8_16 1 +打败它 _8_16 1 +阿杰伊阿卡什 _8_16 1 +Tumi#长袍#尼罗布 _8_16 1 +舒克诺#帕塔尔#努普尔#帕埃 _8_16 1 diff --git a/pom.xml b/pom.xml index 1c012b092..4812b16f1 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ 3.2.4 4.5.1 4.5 - 0.8.4-SNAPSHOT + 0.8.6-SNAPSHOT 2.30.0 @@ -74,6 +74,7 @@ 3.17 0.24.0 42.7.1 + 4.0.8 diff --git a/webapp/packages/chat-sdk/src/components/ChatMsg/Bar/index.tsx b/webapp/packages/chat-sdk/src/components/ChatMsg/Bar/index.tsx index 45d64266d..8f0a5b00b 100644 --- a/webapp/packages/chat-sdk/src/components/ChatMsg/Bar/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatMsg/Bar/index.tsx @@ -1,20 +1,26 @@ import { CHART_BLUE_COLOR, CHART_SECONDARY_COLOR, PREFIX_CLS } from '../../../common/constants'; import { MsgDataType } from '../../../common/type'; -import { getChartLightenColor, getFormattedValue } from '../../../utils/utils'; +import { + formatByDecimalPlaces, + getChartLightenColor, + getFormattedValue, +} from '../../../utils/utils'; import type { ECharts } from 'echarts'; import * as echarts from 'echarts'; import React, { useEffect, useRef, useState } from 'react'; import NoPermissionChart from '../NoPermissionChart'; +import { ColumnType } from '../../../common/type'; import { Spin } from 'antd'; type Props = { data: MsgDataType; triggerResize?: boolean; loading: boolean; + metricField: ColumnType; onApplyAuth?: (model: string) => void; }; -const BarChart: React.FC = ({ data, triggerResize, loading, onApplyAuth }) => { +const BarChart: React.FC = ({ data, triggerResize, loading, metricField, onApplyAuth }) => { const chartRef = useRef(); const [instance, setInstance] = useState(); @@ -70,7 +76,11 @@ const BarChart: React.FC = ({ data, triggerResize, loading, onApplyAuth } }, axisLabel: { formatter: function (value: any) { - return value === 0 ? 0 : getFormattedValue(value); + return value === 0 + ? 0 + : metricField.dataFormatType === 'percent' + ? `${formatByDecimalPlaces(value, metricField.dataFormat?.decimalPlaces || 2)}%` + : getFormattedValue(value); }, }, }, @@ -85,9 +95,17 @@ const BarChart: React.FC = ({ data, triggerResize, loading, onApplyAuth } item.marker } ${ item.seriesName - }${getFormattedValue( - item.value - )}` + }${ + item.value === '' + ? '-' + : metricField.dataFormatType === 'percent' || + metricField.dataFormatType === 'decimal' + ? `${formatByDecimalPlaces( + item.value, + metricField.dataFormat?.decimalPlaces || 2 + )}${metricField.dataFormatType === 'percent' ? '%' : ''}` + : getFormattedValue(item.value) + }` ) .join(''); return `${param.name}
${valueLabels}`; @@ -115,7 +133,11 @@ const BarChart: React.FC = ({ data, triggerResize, loading, onApplyAuth } show: true, position: 'top', formatter: function ({ value }: any) { - return getFormattedValue(value); + return value === 0 + ? 0 + : metricField.dataFormatType === 'percent' + ? `${formatByDecimalPlaces(value, metricField.dataFormat?.decimalPlaces || 2)}%` + : getFormattedValue(value); }, }, data: data.map(item => { diff --git a/webapp/packages/chat-sdk/src/components/ChatMsg/WebPage/index.tsx b/webapp/packages/chat-sdk/src/components/ChatMsg/WebPage/index.tsx index e9ef5917c..f9b7734c1 100644 --- a/webapp/packages/chat-sdk/src/components/ChatMsg/WebPage/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatMsg/WebPage/index.tsx @@ -26,7 +26,6 @@ const WebPage: React.FC = ({ id, data }) => { const { msgId, height } = payload; if (`${msgId}` === `${id}`) { setHeight(height); - // updateMessageContainerScroll(); } return; } diff --git a/webapp/packages/chat-sdk/src/components/ChatMsg/index.tsx b/webapp/packages/chat-sdk/src/components/ChatMsg/index.tsx index 4c1b1ad4f..6930167b1 100644 --- a/webapp/packages/chat-sdk/src/components/ChatMsg/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatMsg/index.tsx @@ -128,13 +128,14 @@ const ChatMsg: React.FC = ({ queryId, data, chartIndex, triggerResize }) if ( categoryField?.length > 0 && metricFields?.length > 0 && - (isMobile ? dataSource?.length <= 20 : dataSource?.length <= 50) + (isMobile ? dataSource?.length <= 5 : dataSource?.length <= 50) ) { return ( ); } diff --git a/webapp/packages/supersonic-fe/package.json b/webapp/packages/supersonic-fe/package.json index 8aa1191c7..df1cc8181 100644 --- a/webapp/packages/supersonic-fe/package.json +++ b/webapp/packages/supersonic-fe/package.json @@ -16,6 +16,7 @@ "dev:inner": "npm run start:dev", "no:dev:os": "NODE_OPTIONS=--openssl-legacy-provider npm run start:osdev", "no:dev:inner": "NODE_OPTIONS=--openssl-legacy-provider npm run start:dev", + "no:build:inner": "cross-env NODE_OPTIONS=--openssl-legacy-provider REACT_APP_ENV=prod APP_TARGET=inner umi build", "gh-pages": "gh-pages -d dist", "i18n-remove": "pro i18n-remove --locale=zh-CN --write", "postinstall": "umi g tmp", diff --git a/webapp/packages/supersonic-fe/src/pages/ChatPlugin/DetailModal.tsx b/webapp/packages/supersonic-fe/src/pages/ChatPlugin/DetailModal.tsx index f5cd87752..e07011eca 100644 --- a/webapp/packages/supersonic-fe/src/pages/ChatPlugin/DetailModal.tsx +++ b/webapp/packages/supersonic-fe/src/pages/ChatPlugin/DetailModal.tsx @@ -10,7 +10,7 @@ import { FunctionParamFormItemType, PluginTypeEnum, } from './type'; -import { getLeafList, traverseTree, uuid } from '@/utils/utils'; +import { traverseTree, uuid } from '@/utils/utils'; import styles from './style.less'; import { PLUGIN_TYPE_MAP } from './constants'; import { DeleteOutlined, PlusOutlined } from '@ant-design/icons'; diff --git a/webapp/packages/supersonic-fe/src/pages/ChatPlugin/index.tsx b/webapp/packages/supersonic-fe/src/pages/ChatPlugin/index.tsx index d386f6d67..5e254989c 100644 --- a/webapp/packages/supersonic-fe/src/pages/ChatPlugin/index.tsx +++ b/webapp/packages/supersonic-fe/src/pages/ChatPlugin/index.tsx @@ -1,4 +1,4 @@ -import { getLeafList } from '@/utils/utils'; +import { getLeafNodes } from '@/utils/utils'; import { PlusOutlined } from '@ant-design/icons'; import { Button, Input, message, Popconfirm, Select, Table, Tag } from 'antd'; import moment from 'moment'; @@ -24,7 +24,7 @@ const PluginManage = () => { const initModelList = async () => { const res = await getModelList(); - setModelList(getLeafList(res.data)); + setModelList(getLeafNodes(res.data)); }; const updateData = async (filters?: any) => { @@ -68,7 +68,7 @@ const PluginManage = () => { return value ? (
{value.map((id) => { - const name = modelList.find((model) => model.id === +id)?.name; + const name = modelList.find((model) => model.id === id)?.name; return name ? {name} : null; })}
diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceCreateForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceCreateForm.tsx index 3bcaf5343..8f26d54d5 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceCreateForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceCreateForm.tsx @@ -5,11 +5,13 @@ import DataSourceFieldForm from './DataSourceFieldForm'; import { formLayout } from '@/components/FormHelper/utils'; import { EnumDataSourceType } from '../constants'; import styles from '../style.less'; -import { updateModel, createModel, getColumns } from '../../service'; +import { updateModel, createModel, getColumns, getUnAvailableItem } from '../../service'; import type { Dispatch } from 'umi'; import type { StateType } from '../../model'; import { connect } from 'umi'; import { ISemantic, IDataSource } from '../../data'; +import { isArrayOfValues } from '@/utils/utils'; +import EffectDimensionAndMetricTipsModal from './EffectDimensionAndMetricTipsModal'; export type CreateFormProps = { domainManger: StateType; @@ -54,6 +56,12 @@ const DataSourceCreateForm: React.FC = ({ const [saveLoading, setSaveLoading] = useState(false); const [hasEmptyNameField, setHasEmptyNameField] = useState(false); const [formDatabaseId, setFormDatabaseId] = useState(); + const [queryParamsState, setQueryParamsState] = useState({}); + const [effectTipsModalOpenState, setEffectTipsModalOpenState] = useState(false); + const [effectTipsData, setEffectTipsData] = useState< + (ISemantic.IDimensionItem | ISemantic.IMetricItem)[] + >([]); + const formValRef = useRef(initFormVal as any); const [form] = Form.useForm(); const { databaseConfigList, selectModelId: modelId, selectDomainId } = domainManger; @@ -84,6 +92,58 @@ const DataSourceCreateForm: React.FC = ({ const forward = () => setCurrentStep(currentStep + 1); const backward = () => setCurrentStep(currentStep - 1); + const checkAvailableItem = async (fields: string[] = []) => { + if (!modelItem) { + return false; + } + const originalFields = modelItem.modelDetail?.fields || []; + const hasRemoveFields = originalFields.reduce( + (fieldList: string[], item: IDataSource.IDataSourceDetailFieldsItem) => { + const { fieldName } = item; + if (!fields.includes(fieldName)) { + fieldList.push(fieldName); + } + return fieldList; + }, + [], + ); + if (!isArrayOfValues(hasRemoveFields)) { + return false; + } + const { code, data, msg } = await getUnAvailableItem({ + modelId: modelItem?.id, + fields: hasRemoveFields, + }); + if (code === 200 && data) { + const { metricResps = [], dimensionResps = [] } = data; + if (!isArrayOfValues(metricResps) && !isArrayOfValues(dimensionResps)) { + return false; + } + setEffectTipsData([...metricResps, ...dimensionResps]); + setEffectTipsModalOpenState(true); + return true; + } + message.error(msg); + return false; + }; + + const saveModel = async (queryParams: any) => { + setSaveLoading(true); + const queryDatasource = isEdit ? updateModel : createModel; + const { code, msg, data } = await queryDatasource(queryParams); + setSaveLoading(false); + if (code === 200) { + message.success('保存模型成功！'); + onSubmit?.({ + ...queryParams, + ...data, + resData: data, + }); + return; + } + message.error(msg); + }; + const getFieldsClassify = (fieldsList: any[]) => { const classify = fieldsList.reduce( (fieldsClassify, item: any) => { @@ -129,6 +189,7 @@ const DataSourceCreateForm: React.FC = ({ case EnumDataSourceType.PRIMARY: fieldsClassify.identifiers.push({ bizName: fieldName, + isCreateDimension, name, type, entityNames, @@ -156,6 +217,7 @@ const DataSourceCreateForm: React.FC = ({ ); return classify; }; + const handleNext = async (saveState: boolean = false) => { const fieldsValue = await form.validateFields(); @@ -175,7 +237,6 @@ const DataSourceCreateForm: React.FC = ({ if (!saveState && currentStep < 1) { forward(); } else { - setSaveLoading(true); const { dbName, tableName } = submitForm; const queryParams = { ...submitForm, @@ -190,19 +251,12 @@ const DataSourceCreateForm: React.FC = ({ sqlQuery: sql, }, }; - const queryDatasource = isEdit ? updateModel : createModel; - const { code, msg, data } = await queryDatasource(queryParams); - setSaveLoading(false); - if (code === 200) { - message.success('保存模型成功！'); - onSubmit?.({ - ...queryParams, - ...data, - resData: data, - }); + setQueryParamsState(queryParams); + const checkState = await checkAvailableItem(fieldColumns.map((item) => item.nameEn)); + if (checkState) { return; } - message.error(msg); + saveModel(queryParams); } }; @@ -484,6 +538,17 @@ const DataSourceCreateForm: React.FC = ({ {renderContent()} {children} + + { + setEffectTipsModalOpenState(false); + }} + onOk={() => { + saveModel(queryParamsState); + }} + /> ); }; diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceFieldForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceFieldForm.tsx index 388059740..e0cb1f698 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceFieldForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceFieldForm.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useState, useEffect } from 'react'; import { Table, Select, Checkbox, Input, Alert, Space, Tooltip, Form, Switch } from 'antd'; import TableTitleTooltips from '../../components/TableTitleTooltips'; import { isUndefined } from 'lodash'; @@ -26,7 +26,7 @@ type FieldItem = { entityNames?: string[]; isTag?: number; }; - +const { Search } = Input; const FormItem = Form.Item; type Props = { @@ -54,6 +54,7 @@ const DataSourceFieldForm: React.FC = ({ fields, sql, onFieldChange, onSq [fieldName]: value, }); }; + const [filterValue, setFliterValue] = useState(); const columns = [ { @@ -296,12 +297,32 @@ const DataSourceFieldForm: React.FC = ({ fields, sql, onFieldChange, onSq }, ]; + const onSearch = (value: any) => { + setFliterValue(value); + }; + + const tableData = filterValue + ? fields.filter((item) => { + return item.bizName.includes(filterValue); + }) || [] + : fields; return ( <> +
+ +
+ - dataSource={fields} + dataSource={tableData} columns={columns} rowKey="bizName" + virtual pagination={false} scroll={{ y: 500 }} /> diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/EffectDimensionAndMetricTipsModal.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/EffectDimensionAndMetricTipsModal.tsx new file mode 100644 index 000000000..87209f908 --- /dev/null +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/EffectDimensionAndMetricTipsModal.tsx @@ -0,0 +1,153 @@ +import React, { useState, useEffect } from 'react'; +import { Form, Input, Modal, Table, Tag, message } from 'antd'; +import TransTypeTag from '../../components/TransTypeTag'; +import { SemanticNodeType } from '../../enum'; +import { ISemantic } from '../../data'; +import styles from '../../components/style.less'; +import { StatusEnum } from '../../enum'; +import { batchUpdateMetricStatus, batchUpdateDimensionStatus } from '../../service'; + +type Props = { + open: boolean; + tableDataSource: (ISemantic.IDimensionItem | ISemantic.IMetricItem)[]; + onOk?: () => void; + onCancel?: () => void; +}; + +const EffectDimensionAndMetricTipsModal: React.FC = ({ + open, + tableDataSource, + onOk, + onCancel, +}) => { + const [loading, setLoading] = useState(false); + useEffect(() => {}, []); + + const queryBatchUpdateDimensionStatus = async (ids: React.Key[], status: StatusEnum) => { + if (Array.isArray(ids) && ids.length === 0) { + return; + } + setLoading(true); + const { code, msg } = await batchUpdateDimensionStatus({ + ids, + status, + }); + setLoading(false); + if (code === 200) { + return; + } + message.error(msg); + }; + + const queryBatchUpdateMetricStatus = async (ids: React.Key[], status: StatusEnum) => { + if (Array.isArray(ids) && ids.length === 0) { + return; + } + const { code, msg } = await batchUpdateMetricStatus({ + ids, + status, + }); + if (code === 200) { + return; + } + message.error(msg); + }; + + const updateState = async () => { + const dimensionIds: React.Key[] = []; + const metricIds: React.Key[] = []; + tableDataSource.forEach((item) => { + if (item.typeEnum === SemanticNodeType.DIMENSION) { + dimensionIds.push(item.id); + } + if (item.typeEnum === SemanticNodeType.METRIC) { + metricIds.push(item.id); + } + }); + if (dimensionIds.length > 0) { + await queryBatchUpdateDimensionStatus(dimensionIds, StatusEnum.UNAVAILABLE); + } + if (metricIds.length > 0) { + await queryBatchUpdateMetricStatus(metricIds, StatusEnum.UNAVAILABLE); + } + onOk?.(); + }; + + const columns = [ + { + title: '名称', + dataIndex: 'name', + width: 100, + }, + { + title: '英文名称', + dataIndex: 'bizName', + width: 100, + }, + { + title: '类型', + dataIndex: 'typeEnum', + width: 80, + render: (transType: SemanticNodeType) => { + return ; + }, + }, + { + title: '创建人', + width: 100, + dataIndex: 'createdBy', + }, + ]; + + return ( + { + updateState(); + }} + // footer={renderFooter()} + onCancel={() => { + onCancel?.(); + }} + > +

+ 检测到模型信息变更会对以下 + + 指标 + + 和 + + 维度 + + 产生影响。如确认保存，将会自动置为 + + 不可用状态 + + 。 +

+ + + ); +}; + +export default EffectDimensionAndMetricTipsModal; diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/View/components/ViewCreateFormModal.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/View/components/ViewCreateFormModal.tsx index a9b32f37f..682658051 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/View/components/ViewCreateFormModal.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/View/components/ViewCreateFormModal.tsx @@ -131,9 +131,9 @@ const ViewCreateFormModal: React.FC = ({ - + */}