[improvement][headless]Remove fixed TimeEnums fields.

This commit is contained in:
jerryjzhang
2024-12-12 00:27:09 +08:00
parent a8868ba9ab
commit 9f8793bfe2
7 changed files with 96 additions and 49 deletions

View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.common.pojo.enums; package com.tencent.supersonic.common.pojo.enums;
public enum TimeDimensionEnum { public enum TimeDimensionEnum {
DAY, DAY, WEEK, MONTH;
WEEK,
MONTH;
} }

View File

@@ -126,19 +126,20 @@ public class DateModeUtils {
return ImmutablePair.of(start, dateDate.getEndDate()); return ImmutablePair.of(start, dateDate.getEndDate());
} }
public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr, DateConf dateInfo) { public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr,
DateConf dateInfo) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr); DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
String endStr = endData.format(formatter); String endStr = endData.format(formatter);
String start = endData.minusMonths(unit).format(formatter); String start = endData.minusMonths(unit).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), start, dateInfo.getDateField(), return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), start,
endStr); dateInfo.getDateField(), endStr);
} }
public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) { public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) {
List<ImmutablePair<String, String>> range = recentMonth(dateDate, dateInfo); List<ImmutablePair<String, String>> range = recentMonth(dateDate, dateInfo);
if (range.size() == 1) { if (range.size() == 1) {
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), range.get(0).left, return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateInfo.getDateField(), range.get(0).right); range.get(0).left, dateInfo.getDateField(), range.get(0).right);
} }
if (range.size() > 0) { if (range.size() > 0) {
StringJoiner joiner = new StringJoiner(","); StringJoiner joiner = new StringJoiner(",");
@@ -176,8 +177,8 @@ public class DateModeUtils {
public String recentWeekStr(LocalDate endData, Long unit, DateConf dataInfo) { public String recentWeekStr(LocalDate endData, Long unit, DateConf dataInfo) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT); DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT);
String start = endData.minusDays(unit * 7).format(formatter); String start = endData.minusDays(unit * 7).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", dataInfo.getDateField(), start, dataInfo.getDateField(), return String.format("(%s >= '%s' and %s <= '%s')", dataInfo.getDateField(), start,
endData.format(formatter)); dataInfo.getDateField(), endData.format(formatter));
} }
public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) { public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) {

View File

@@ -2,7 +2,10 @@ package com.tencent.supersonic.headless.chat.parser.llm;
import com.tencent.supersonic.common.jsqlparser.SqlValidHelper; import com.tencent.supersonic.common.jsqlparser.SqlValidHelper;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Text2SQLExemplar; import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.QueryManager; import com.tencent.supersonic.headless.chat.query.QueryManager;
@@ -50,6 +53,15 @@ public class LLMResponseService {
parseInfo.setQueryMode(semanticQuery.getQueryMode()); parseInfo.setQueryMode(semanticQuery.getQueryMode());
parseInfo.getSqlInfo().setParsedS2SQL(s2SQL); parseInfo.getSqlInfo().setParsedS2SQL(s2SQL);
parseInfo.getSqlInfo().setCorrectedS2SQL(s2SQL); parseInfo.getSqlInfo().setCorrectedS2SQL(s2SQL);
DataSetSchema dataSetSchema =
queryCtx.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
if (Objects.nonNull(partitionDimension)) {
DateConf dateConf = new DateConf();
dateConf.setDateField(partitionDimension.getName());
parseInfo.setDateInfo(dateConf);
}
queryCtx.getCandidateQueries().add(semanticQuery); queryCtx.getCandidateQueries().add(semanticQuery);
} }

View File

@@ -100,6 +100,8 @@ public class PromptHelper {
} }
if (StringUtils.isNotEmpty(metric.getDefaultAgg())) { if (StringUtils.isNotEmpty(metric.getDefaultAgg())) {
metricStr.append(" AGGREGATE '" + metric.getDefaultAgg().toUpperCase() + "'"); metricStr.append(" AGGREGATE '" + metric.getDefaultAgg().toUpperCase() + "'");
} else {
metricStr.append(" AGGREGATE 'NONE'");
} }
metricStr.append(">"); metricStr.append(">");
metrics.add(metricStr.toString()); metrics.add(metricStr.toString());

View File

@@ -40,8 +40,8 @@ public class DefaultDimValueConverter implements QueryConverter {
return; return;
} }
String sql = queryStatement.getSqlQueryParam().getSql(); String sql = queryStatement.getSqlQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream() List<String> whereFields =
.collect(Collectors.toList()); SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) { if (!CollectionUtils.isEmpty(whereFields)) {
return; return;
} }

View File

@@ -8,23 +8,58 @@ import com.github.pagehelper.PageInfo;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.*; import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum; import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper; import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.pojo.*; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.service.*; import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -37,7 +72,18 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Service @Service
@@ -656,38 +702,24 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
&& !metricResp.getDefaultAgg().isEmpty())) { && !metricResp.getDefaultAgg().isEmpty())) {
return metricResp.getDefaultAgg(); return metricResp.getDefaultAgg();
} }
// FIELD define will get from expr
if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) {
return SqlSelectFunctionHelper.getFirstAggregateFunctions(metricResp.getExpr());
}
// METRIC define will get from first metric
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
if (!CollectionUtils.isEmpty(metricResp.getMetricDefineByMetricParams().getMetrics())) {
MetricParam metricParam =
metricResp.getMetricDefineByMetricParams().getMetrics().get(0);
MetricResp firstMetricResp =
getMetric(modelResp.getDomainId(), metricParam.getBizName());
if (Objects.nonNull(firstMetricResp)) {
return getDefaultAgg(firstMetricResp, modelResp);
}
return "";
}
}
// Measure define will get from first measure // Measure define will get from first measure
List<Measure> measures = modelResp.getModelDetail().getMeasures(); if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
List<MeasureParam> measureParams = List<Measure> measures = modelResp.getModelDetail().getMeasures();
metricResp.getMetricDefineByMeasureParams().getMeasures(); List<MeasureParam> measureParams =
if (CollectionUtils.isEmpty(measureParams)) { metricResp.getMetricDefineByMeasureParams().getMeasures();
return ""; if (CollectionUtils.isEmpty(measureParams)) {
} return null;
MeasureParam firstMeasure = measureParams.get(0); }
MeasureParam firstMeasure = measureParams.get(0);
for (Measure measure : measures) { for (Measure measure : measures) {
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) { if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
return measure.getAgg(); return measure.getAgg();
}
} }
} }
return "";
return null;
} }
@Override @Override

View File

@@ -39,6 +39,7 @@ public class MetricTest extends BaseTest {
} }
@Test @Test
@SetSystemProperty(key = "s2.test", value = "true")
public void testMetricModel() throws Exception { public void testMetricModel() throws Exception {
QueryResult actualResult = submitNewChat("超音数 访问次数", agent.getId()); QueryResult actualResult = submitNewChat("超音数 访问次数", agent.getId());
@@ -56,6 +57,7 @@ public class MetricTest extends BaseTest {
assertQueryResult(expectedResult, actualResult); assertQueryResult(expectedResult, actualResult);
assert actualResult.getQueryResults().size() == 1; assert actualResult.getQueryResults().size() == 1;
assert actualResult.getQuerySql().contains("s2_pv_uv_statis");
} }
@Test @Test