mirror of
https://github.com/tencentmusic/supersonic.git
synced 2026-06-26 06:39:20 +08:00
feat(data): 添加时间戳数据类型支持并优化维度转换逻辑
- 新增 TIMESTAMP 数据类型枚举值 - 为整型数据类型匹配添加 INT 关键字识别 - 移除维度转换中的冗余数据类型设置逻辑 - 修复数据类型转换过程中的潜在问题
This commit is contained in:
@@ -26,12 +26,10 @@ public class SqlDialectFactory {
|
|||||||
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
||||||
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||||
public static final Context PRESTO_CONTEXT =
|
public static final Context PRESTO_CONTEXT = SqlDialect.EMPTY_CONTEXT
|
||||||
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.PRESTO)
|
.withDatabaseProduct(DatabaseProduct.PRESTO).withLiteralQuoteString("'")
|
||||||
.withLiteralQuoteString("'")
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
.withLiteralEscapedQuoteString("''")
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||||
.withUnquotedCasing(Casing.UNCHANGED)
|
|
||||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
|
||||||
public static final Context KYUUBI_CONTEXT =
|
public static final Context KYUUBI_CONTEXT =
|
||||||
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
|
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
|
||||||
.withLiteralQuoteString("'").withIdentifierQuoteString("`")
|
.withLiteralQuoteString("'").withIdentifierQuoteString("`")
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ public enum DataTypeEnums {
|
|||||||
|
|
||||||
DATE("DATE"),
|
DATE("DATE"),
|
||||||
|
|
||||||
|
TIMESTAMP("TIMESTAMP"),
|
||||||
|
|
||||||
BIGINT("BIGINT"),
|
BIGINT("BIGINT"),
|
||||||
|
|
||||||
INT("INT"),
|
INT("INT"),
|
||||||
@@ -39,6 +41,9 @@ public enum DataTypeEnums {
|
|||||||
return typeEnum;
|
return typeEnum;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (type != null && type.toUpperCase().contains("INT")) {
|
||||||
|
return DataTypeEnums.INT;
|
||||||
|
}
|
||||||
return DataTypeEnums.UNKNOWN;
|
return DataTypeEnums.UNKNOWN;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ public class ChatQueryContext implements Serializable {
|
|||||||
candidateQueries = candidateQueries.stream()
|
candidateQueries = candidateQueries.stream()
|
||||||
.sorted(Comparator.comparing(
|
.sorted(Comparator.comparing(
|
||||||
semanticQuery -> semanticQuery.getParseInfo().getScore(),
|
semanticQuery -> semanticQuery.getParseInfo().getScore(),
|
||||||
Comparator.reverseOrder())).collect(Collectors.toList());
|
Comparator.reverseOrder()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
return candidateQueries;
|
return candidateQueries;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -79,11 +79,11 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
|||||||
public DataSetResp getDataSet(Long id) {
|
public DataSetResp getDataSet(Long id) {
|
||||||
DataSetDO dataSetDO = getById(id);
|
DataSetDO dataSetDO = getById(id);
|
||||||
DataSetResp dataSetResp = convert(dataSetDO);
|
DataSetResp dataSetResp = convert(dataSetDO);
|
||||||
|
|
||||||
if (dataSetResp.getDataSetDetail() != null) {
|
if (dataSetResp.getDataSetDetail() != null) {
|
||||||
expandIncludesAllModels(dataSetResp);
|
expandIncludesAllModels(dataSetResp);
|
||||||
}
|
}
|
||||||
|
|
||||||
return dataSetResp;
|
return dataSetResp;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -286,10 +286,8 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Set<Long> includeAllModelIds = configs.stream()
|
Set<Long> includeAllModelIds = configs.stream().filter(DataSetModelConfig::getIncludesAll)
|
||||||
.filter(DataSetModelConfig::getIncludesAll)
|
.map(DataSetModelConfig::getId).collect(Collectors.toSet());
|
||||||
.map(DataSetModelConfig::getId)
|
|
||||||
.collect(Collectors.toSet());
|
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(includeAllModelIds)) {
|
if (CollectionUtils.isEmpty(includeAllModelIds)) {
|
||||||
return;
|
return;
|
||||||
@@ -302,28 +300,26 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
|||||||
List<DimensionResp> allDimensions = dimensionService.getDimensions(metaFilter);
|
List<DimensionResp> allDimensions = dimensionService.getDimensions(metaFilter);
|
||||||
List<MetricResp> allMetrics = metricService.getMetrics(metaFilter);
|
List<MetricResp> allMetrics = metricService.getMetrics(metaFilter);
|
||||||
|
|
||||||
Map<Long, List<Long>> modelDimensionMap = allDimensions.stream()
|
Map<Long, List<Long>> modelDimensionMap =
|
||||||
.collect(Collectors.groupingBy(
|
allDimensions.stream().collect(Collectors.groupingBy(DimensionResp::getModelId,
|
||||||
DimensionResp::getModelId,
|
Collectors.mapping(DimensionResp::getId, Collectors.toList())));
|
||||||
Collectors.mapping(DimensionResp::getId, Collectors.toList())
|
|
||||||
));
|
|
||||||
|
|
||||||
Map<Long, List<Long>> modelMetricMap = allMetrics.stream()
|
Map<Long, List<Long>> modelMetricMap =
|
||||||
.collect(Collectors.groupingBy(
|
allMetrics.stream().collect(Collectors.groupingBy(MetricResp::getModelId,
|
||||||
MetricResp::getModelId,
|
Collectors.mapping(MetricResp::getId, Collectors.toList())));
|
||||||
Collectors.mapping(MetricResp::getId, Collectors.toList())
|
|
||||||
));
|
|
||||||
|
|
||||||
for (DataSetModelConfig config : configs) {
|
for (DataSetModelConfig config : configs) {
|
||||||
if (Boolean.TRUE.equals(config.getIncludesAll())) {
|
if (Boolean.TRUE.equals(config.getIncludesAll())) {
|
||||||
Long modelId = config.getId();
|
Long modelId = config.getId();
|
||||||
|
|
||||||
List<Long> modelDimensions = modelDimensionMap.getOrDefault(modelId, Lists.newArrayList());
|
List<Long> modelDimensions =
|
||||||
|
modelDimensionMap.getOrDefault(modelId, Lists.newArrayList());
|
||||||
Set<Long> existingDimensions = new HashSet<>(config.getDimensions());
|
Set<Long> existingDimensions = new HashSet<>(config.getDimensions());
|
||||||
existingDimensions.addAll(modelDimensions);
|
existingDimensions.addAll(modelDimensions);
|
||||||
config.setDimensions(new ArrayList<>(existingDimensions));
|
config.setDimensions(new ArrayList<>(existingDimensions));
|
||||||
|
|
||||||
List<Long> modelMetrics = modelMetricMap.getOrDefault(modelId, Lists.newArrayList());
|
List<Long> modelMetrics =
|
||||||
|
modelMetricMap.getOrDefault(modelId, Lists.newArrayList());
|
||||||
Set<Long> existingMetrics = new HashSet<>(config.getMetrics());
|
Set<Long> existingMetrics = new HashSet<>(config.getMetrics());
|
||||||
existingMetrics.addAll(modelMetrics);
|
existingMetrics.addAll(modelMetrics);
|
||||||
config.setMetrics(new ArrayList<>(existingMetrics));
|
config.setMetrics(new ArrayList<>(existingMetrics));
|
||||||
|
|||||||
@@ -73,9 +73,6 @@ public class DimensionConverter {
|
|||||||
if (dimensionReq.getTypeParams() != null) {
|
if (dimensionReq.getTypeParams() != null) {
|
||||||
dimensionDO.setTypeParams(JSONObject.toJSONString(dimensionReq.getTypeParams()));
|
dimensionDO.setTypeParams(JSONObject.toJSONString(dimensionReq.getTypeParams()));
|
||||||
}
|
}
|
||||||
if (dimensionReq.getDataType() != null) {
|
|
||||||
dimensionDO.setDataType(dimensionReq.getDataType().getType());
|
|
||||||
}
|
|
||||||
dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
|
dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
|
||||||
return dimensionDO;
|
return dimensionDO;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user