mirror of
https://github.com/tencentmusic/supersonic.git
synced 2026-06-26 06:39:20 +08:00
Compare commits
5 Commits
c0af25fe2e
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de60be3908 | ||
|
|
0868850edd | ||
|
|
6d41ce4c5b | ||
|
|
ef2f07a59e | ||
|
|
c3cf8b1889 |
@@ -26,12 +26,10 @@ public class SqlDialectFactory {
|
|||||||
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
||||||
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||||
public static final Context PRESTO_CONTEXT =
|
public static final Context PRESTO_CONTEXT = SqlDialect.EMPTY_CONTEXT
|
||||||
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.PRESTO)
|
.withDatabaseProduct(DatabaseProduct.PRESTO).withLiteralQuoteString("'")
|
||||||
.withLiteralQuoteString("'")
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
.withLiteralEscapedQuoteString("''")
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||||
.withUnquotedCasing(Casing.UNCHANGED)
|
|
||||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
|
||||||
public static final Context KYUUBI_CONTEXT =
|
public static final Context KYUUBI_CONTEXT =
|
||||||
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
|
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
|
||||||
.withLiteralQuoteString("'").withIdentifierQuoteString("`")
|
.withLiteralQuoteString("'").withIdentifierQuoteString("`")
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ public enum DataTypeEnums {
|
|||||||
|
|
||||||
DATE("DATE"),
|
DATE("DATE"),
|
||||||
|
|
||||||
|
TIMESTAMP("TIMESTAMP"),
|
||||||
|
|
||||||
BIGINT("BIGINT"),
|
BIGINT("BIGINT"),
|
||||||
|
|
||||||
INT("INT"),
|
INT("INT"),
|
||||||
@@ -39,6 +41,9 @@ public enum DataTypeEnums {
|
|||||||
return typeEnum;
|
return typeEnum;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (type != null && type.toUpperCase().contains("INT")) {
|
||||||
|
return DataTypeEnums.INT;
|
||||||
|
}
|
||||||
return DataTypeEnums.UNKNOWN;
|
return DataTypeEnums.UNKNOWN;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ public class ChatQueryContext implements Serializable {
|
|||||||
candidateQueries = candidateQueries.stream()
|
candidateQueries = candidateQueries.stream()
|
||||||
.sorted(Comparator.comparing(
|
.sorted(Comparator.comparing(
|
||||||
semanticQuery -> semanticQuery.getParseInfo().getScore(),
|
semanticQuery -> semanticQuery.getParseInfo().getScore(),
|
||||||
Comparator.reverseOrder())).collect(Collectors.toList());
|
Comparator.reverseOrder()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
return candidateQueries;
|
return candidateQueries;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -50,8 +51,12 @@ public class HanlpDictMatchStrategy extends SingleMatchStrategy<HanlpMapResult>
|
|||||||
return new ArrayList<>();
|
return new ArrayList<>();
|
||||||
}
|
}
|
||||||
// step3. merge pre/suffix result
|
// step3. merge pre/suffix result
|
||||||
|
// sort by similarity (desc) first, then name length (desc), so that
|
||||||
|
// higher-similarity records are inserted first and survive LinkedHashSet dedup
|
||||||
hanlpMapResults = hanlpMapResults.stream()
|
hanlpMapResults = hanlpMapResults.stream()
|
||||||
.sorted((a, b) -> -(b.getName().length() - a.getName().length()))
|
.sorted(Comparator.comparingDouble(HanlpMapResult::getSimilarity).reversed()
|
||||||
|
.thenComparing((a, b) -> Integer.compare(b.getName().length(),
|
||||||
|
a.getName().length())))
|
||||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
|
|
||||||
// step4. filter by similarity
|
// step4. filter by similarity
|
||||||
|
|||||||
@@ -123,15 +123,9 @@ public class MapFilter {
|
|||||||
.filter(SchemaElementMatch::isFullMatched).collect(Collectors.toList());
|
.filter(SchemaElementMatch::isFullMatched).collect(Collectors.toList());
|
||||||
|
|
||||||
if (!fullMatches.isEmpty()) {
|
if (!fullMatches.isEmpty()) {
|
||||||
// If there are objects with similarity=1.0, choose the one with the longest
|
// Keep all records with similarity=1.0, as they may correspond to different
|
||||||
// detectWord and smallest offset
|
// elementIds with the same detectWord
|
||||||
SchemaElementMatch bestMatch = fullMatches.stream()
|
result.addAll(fullMatches);
|
||||||
.max(Comparator.comparing(
|
|
||||||
(SchemaElementMatch match) -> match.getDetectWord().length()))
|
|
||||||
.orElse(null);
|
|
||||||
if (bestMatch != null) {
|
|
||||||
result.add(bestMatch);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// If there are no objects with similarity=1.0, keep all objects with similarity<1.0
|
// If there are no objects with similarity=1.0, keep all objects with similarity<1.0
|
||||||
result.addAll(group);
|
result.addAll(group);
|
||||||
|
|||||||
@@ -79,11 +79,11 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
|||||||
public DataSetResp getDataSet(Long id) {
|
public DataSetResp getDataSet(Long id) {
|
||||||
DataSetDO dataSetDO = getById(id);
|
DataSetDO dataSetDO = getById(id);
|
||||||
DataSetResp dataSetResp = convert(dataSetDO);
|
DataSetResp dataSetResp = convert(dataSetDO);
|
||||||
|
|
||||||
if (dataSetResp.getDataSetDetail() != null) {
|
if (dataSetResp.getDataSetDetail() != null) {
|
||||||
expandIncludesAllModels(dataSetResp);
|
expandIncludesAllModels(dataSetResp);
|
||||||
}
|
}
|
||||||
|
|
||||||
return dataSetResp;
|
return dataSetResp;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -286,10 +286,8 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Set<Long> includeAllModelIds = configs.stream()
|
Set<Long> includeAllModelIds = configs.stream().filter(DataSetModelConfig::getIncludesAll)
|
||||||
.filter(DataSetModelConfig::getIncludesAll)
|
.map(DataSetModelConfig::getId).collect(Collectors.toSet());
|
||||||
.map(DataSetModelConfig::getId)
|
|
||||||
.collect(Collectors.toSet());
|
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(includeAllModelIds)) {
|
if (CollectionUtils.isEmpty(includeAllModelIds)) {
|
||||||
return;
|
return;
|
||||||
@@ -302,28 +300,26 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
|||||||
List<DimensionResp> allDimensions = dimensionService.getDimensions(metaFilter);
|
List<DimensionResp> allDimensions = dimensionService.getDimensions(metaFilter);
|
||||||
List<MetricResp> allMetrics = metricService.getMetrics(metaFilter);
|
List<MetricResp> allMetrics = metricService.getMetrics(metaFilter);
|
||||||
|
|
||||||
Map<Long, List<Long>> modelDimensionMap = allDimensions.stream()
|
Map<Long, List<Long>> modelDimensionMap =
|
||||||
.collect(Collectors.groupingBy(
|
allDimensions.stream().collect(Collectors.groupingBy(DimensionResp::getModelId,
|
||||||
DimensionResp::getModelId,
|
Collectors.mapping(DimensionResp::getId, Collectors.toList())));
|
||||||
Collectors.mapping(DimensionResp::getId, Collectors.toList())
|
|
||||||
));
|
|
||||||
|
|
||||||
Map<Long, List<Long>> modelMetricMap = allMetrics.stream()
|
Map<Long, List<Long>> modelMetricMap =
|
||||||
.collect(Collectors.groupingBy(
|
allMetrics.stream().collect(Collectors.groupingBy(MetricResp::getModelId,
|
||||||
MetricResp::getModelId,
|
Collectors.mapping(MetricResp::getId, Collectors.toList())));
|
||||||
Collectors.mapping(MetricResp::getId, Collectors.toList())
|
|
||||||
));
|
|
||||||
|
|
||||||
for (DataSetModelConfig config : configs) {
|
for (DataSetModelConfig config : configs) {
|
||||||
if (Boolean.TRUE.equals(config.getIncludesAll())) {
|
if (Boolean.TRUE.equals(config.getIncludesAll())) {
|
||||||
Long modelId = config.getId();
|
Long modelId = config.getId();
|
||||||
|
|
||||||
List<Long> modelDimensions = modelDimensionMap.getOrDefault(modelId, Lists.newArrayList());
|
List<Long> modelDimensions =
|
||||||
|
modelDimensionMap.getOrDefault(modelId, Lists.newArrayList());
|
||||||
Set<Long> existingDimensions = new HashSet<>(config.getDimensions());
|
Set<Long> existingDimensions = new HashSet<>(config.getDimensions());
|
||||||
existingDimensions.addAll(modelDimensions);
|
existingDimensions.addAll(modelDimensions);
|
||||||
config.setDimensions(new ArrayList<>(existingDimensions));
|
config.setDimensions(new ArrayList<>(existingDimensions));
|
||||||
|
|
||||||
List<Long> modelMetrics = modelMetricMap.getOrDefault(modelId, Lists.newArrayList());
|
List<Long> modelMetrics =
|
||||||
|
modelMetricMap.getOrDefault(modelId, Lists.newArrayList());
|
||||||
Set<Long> existingMetrics = new HashSet<>(config.getMetrics());
|
Set<Long> existingMetrics = new HashSet<>(config.getMetrics());
|
||||||
existingMetrics.addAll(modelMetrics);
|
existingMetrics.addAll(modelMetrics);
|
||||||
config.setMetrics(new ArrayList<>(existingMetrics));
|
config.setMetrics(new ArrayList<>(existingMetrics));
|
||||||
|
|||||||
@@ -123,8 +123,10 @@ public class DataSetSchemaBuilder {
|
|||||||
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE,
|
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE,
|
||||||
dim.getDataType());
|
dim.getDataType());
|
||||||
} else {
|
} else {
|
||||||
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE,
|
if (dataTypeMap.containsKey(dim.getModelId())) {
|
||||||
dataTypeMap.get(dim.getModelId()).get(dim.getBizName()));
|
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE,
|
||||||
|
dataTypeMap.get(dim.getModelId()).get(dim.getBizName()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (dim.isTimeDimension()) {
|
if (dim.isTimeDimension()) {
|
||||||
String timeFormat =
|
String timeFormat =
|
||||||
|
|||||||
@@ -73,9 +73,6 @@ public class DimensionConverter {
|
|||||||
if (dimensionReq.getTypeParams() != null) {
|
if (dimensionReq.getTypeParams() != null) {
|
||||||
dimensionDO.setTypeParams(JSONObject.toJSONString(dimensionReq.getTypeParams()));
|
dimensionDO.setTypeParams(JSONObject.toJSONString(dimensionReq.getTypeParams()));
|
||||||
}
|
}
|
||||||
if (dimensionReq.getDataType() != null) {
|
|
||||||
dimensionDO.setDataType(dimensionReq.getDataType().getType());
|
|
||||||
}
|
|
||||||
dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
|
dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
|
||||||
return dimensionDO;
|
return dimensionDO;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
<result column="status" jdbcType="INTEGER" property="status"/>
|
<result column="status" jdbcType="INTEGER" property="status"/>
|
||||||
<result column="sensitive_level" jdbcType="INTEGER" property="sensitiveLevel"/>
|
<result column="sensitive_level" jdbcType="INTEGER" property="sensitiveLevel"/>
|
||||||
<result column="type" jdbcType="VARCHAR" property="type"/>
|
<result column="type" jdbcType="VARCHAR" property="type"/>
|
||||||
|
<result column="data_type" jdbcType="VARCHAR" property="dataType"/>
|
||||||
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt"/>
|
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt"/>
|
||||||
<result column="created_by" jdbcType="VARCHAR" property="createdBy"/>
|
<result column="created_by" jdbcType="VARCHAR" property="createdBy"/>
|
||||||
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt"/>
|
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt"/>
|
||||||
@@ -67,14 +68,16 @@
|
|||||||
<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
|
<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
|
||||||
insert into s2_dimension (name, biz_name,
|
insert into s2_dimension (name, biz_name,
|
||||||
description, status, model_id,
|
description, status, model_id,
|
||||||
type, type_params, expr,created_at, created_by,
|
type, data_type, type_params, expr,created_at, created_by,
|
||||||
updated_by, updated_at, semantic_type,sensitive_level, is_tag, ext)
|
updated_by, updated_at, semantic_type,sensitive_level, is_tag, ext)
|
||||||
values
|
values
|
||||||
<foreach collection="list" item="dimension" separator=",">
|
<foreach collection="list" item="dimension" separator=",">
|
||||||
(#{dimension.name,jdbcType=VARCHAR}, #{dimension.bizName,jdbcType=VARCHAR},
|
(#{dimension.name,jdbcType=VARCHAR}, #{dimension.bizName,jdbcType=VARCHAR},
|
||||||
#{dimension.description,jdbcType=VARCHAR}, #{dimension.status,jdbcType=INTEGER},
|
#{dimension.description,jdbcType=VARCHAR}, #{dimension.status,jdbcType=INTEGER},
|
||||||
#{dimension.modelId,jdbcType=BIGINT},
|
#{dimension.modelId,jdbcType=BIGINT},
|
||||||
#{dimension.type,jdbcType=VARCHAR}, #{dimension.typeParams,jdbcType=VARCHAR},
|
#{dimension.type,jdbcType=VARCHAR},
|
||||||
|
#{dimension.dataType,jdbcType=VARCHAR},
|
||||||
|
#{dimension.typeParams,jdbcType=VARCHAR},
|
||||||
#{dimension.expr,jdbcType=VARCHAR}, #{dimension.createdAt,jdbcType=TIMESTAMP},
|
#{dimension.expr,jdbcType=VARCHAR}, #{dimension.createdAt,jdbcType=TIMESTAMP},
|
||||||
#{dimension.createdBy,jdbcType=VARCHAR},
|
#{dimension.createdBy,jdbcType=VARCHAR},
|
||||||
#{dimension.updatedBy,jdbcType=VARCHAR}, #{dimension.updatedAt,jdbcType=TIMESTAMP},
|
#{dimension.updatedBy,jdbcType=VARCHAR}, #{dimension.updatedAt,jdbcType=TIMESTAMP},
|
||||||
@@ -98,6 +101,9 @@
|
|||||||
<if test="dimension.status != null">status = #{dimension.status,jdbcType=INTEGER},</if>
|
<if test="dimension.status != null">status = #{dimension.status,jdbcType=INTEGER},</if>
|
||||||
<if test="dimension.modelId != null">model_id = #{dimension.modelId,jdbcType=BIGINT},</if>
|
<if test="dimension.modelId != null">model_id = #{dimension.modelId,jdbcType=BIGINT},</if>
|
||||||
<if test="dimension.type != null and dimension.type !=''">type = #{dimension.type,jdbcType=VARCHAR},</if>
|
<if test="dimension.type != null and dimension.type !=''">type = #{dimension.type,jdbcType=VARCHAR},</if>
|
||||||
|
<if test="dimension.dataType != null and dimension.dataType !=''">data_type =
|
||||||
|
#{dimension.dataType,jdbcType=VARCHAR},
|
||||||
|
</if>
|
||||||
<if test="dimension.typeParams != null and dimension.typeParams !=''">type_params =
|
<if test="dimension.typeParams != null and dimension.typeParams !=''">type_params =
|
||||||
#{dimension.typeParams,jdbcType=VARCHAR},
|
#{dimension.typeParams,jdbcType=VARCHAR},
|
||||||
</if>
|
</if>
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ const ModelFieldForm: React.FC<Props> = ({
|
|||||||
let defaultParams:any = {};
|
let defaultParams:any = {};
|
||||||
if (value === EnumDataSourceType.MEASURES) {
|
if (value === EnumDataSourceType.MEASURES) {
|
||||||
defaultParams = {
|
defaultParams = {
|
||||||
agg: AGG_OPTIONS[0].value,
|
agg: AGG_OPTIONS[AGG_OPTIONS.length - 1].value,
|
||||||
classType: EnumDataSourceType.MEASURES,
|
classType: EnumDataSourceType.MEASURES,
|
||||||
type: EnumDataSourceType.MEASURES,
|
type: EnumDataSourceType.MEASURES,
|
||||||
};
|
};
|
||||||
@@ -217,7 +217,7 @@ const ModelFieldForm: React.FC<Props> = ({
|
|||||||
handleFieldChange(record, 'agg', value);
|
handleFieldChange(record, 'agg', value);
|
||||||
}}
|
}}
|
||||||
allowClear
|
allowClear
|
||||||
defaultValue={AGG_OPTIONS[0].value}
|
defaultValue={AGG_OPTIONS[AGG_OPTIONS.length - 1].value}
|
||||||
style={{ width: '100%' }}
|
style={{ width: '100%' }}
|
||||||
>
|
>
|
||||||
{AGG_OPTIONS.map((item) => (
|
{AGG_OPTIONS.map((item) => (
|
||||||
|
|||||||
Reference in New Issue
Block a user