topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Remove the platform from the Docker Compose file and ensure the existence of a field with the same name before deleting a specific data date. (#1609)

Browse Source
This commit is contained in:
lexluo09
2024-08-28 15:22:54 +08:00
committed by GitHub
parent 0fc4713c4f
commit c4284f9784
4 changed files with 33 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docker/docker-compose.yml
View File

@@ -1,7 +1,6 @@
services:
chroma:
image: chromadb/chroma:0.5.3
platform: linux/amd64
privileged: true
container_name: supersonic_chroma
ports:
@@ -23,7 +22,6 @@ services:
mysql:
image: mysql:8.0
privileged: true
platform: linux/amd64
container_name: supersonic_mysql
environment:
LANG: 'C.UTF-8' # 设置环境变量
@@ -51,9 +49,8 @@ services:
retries: 5
db_init:
image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-0.9.6}
image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-latest}
privileged: true
platform: linux/amd64
container_name: supersonic_db_init
depends_on:
mysql:
@@ -64,8 +61,8 @@ services:
sh -c "
sleep 15 &&
if ! mysql -h supersonic_mysql -usupersonic_user -psupersonic_password -e 'use supersonic_db; show tables;' | grep -q 's2_database'; then
mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-0.9.6}/conf/db/schema-mysql.sql &&
mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-0.9.6}/conf/db/data-mysql.sql
mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql.sql &&
mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql.sql
else
echo 'Database already initialized.'
fi
@@ -76,9 +73,8 @@ services:
- 8.8.4.4
supersonic_standalone:
image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-0.9.6}
image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-latest}
privileged: true
platform: linux/amd64
container_name: supersonic_standalone
environment:
DB_HOST: supersonic_mysql
@@ -103,9 +99,9 @@ services:
- 8.8.4.4
volumes:
#1.Named Volumes are best for persistent data managed by Docker.
- supersonic_data:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-0.9.6}
- supersonic_data:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}
#2.Bind Mounts are suitable for frequent modifications and debugging.
# - ./conf/langchain4j-prd.yaml:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/langchain4j-prd.yaml
# - ./conf/application-prd.yaml:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/application-prd.yaml
#3.Detailed Bind Mounts offer more control over the mount behavior.
# - type: bind
# source: ./supersonic-standalone-${SUPERSONIC_VERSION:-latest}/logs

41
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java
View File

@@ -45,6 +45,20 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
protected Map<String, String> getFieldNameMap(ChatQueryContext chatQueryContext, Long dataSetId) {
Map<String, String> result = getFieldNameMapFromDB(chatQueryContext, dataSetId);
if (chatQueryContext.containsPartitionDimensions(dataSetId)) {
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
}
return result;
}
private static Map<String, String> getFieldNameMapFromDB(ChatQueryContext chatQueryContext, Long dataSetId) {
SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema();
List<SchemaElement> dbAllFields = new ArrayList<>();
@@ -52,7 +66,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
dbAllFields.addAll(semanticSchema.getDimensions());
// support fieldName and field alias
Map<String, String> result = dbAllFields.stream()
return dbAllFields.stream()
.filter(entry -> dataSetId.equals(entry.getDataSetId()))
.flatMap(schemaElement -> {
Set<String> elements = new HashSet<>();
@@ -63,14 +77,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return elements.stream();
})
.collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1));
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
return result;
}
protected void addAggregateToMetric(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
@@ -131,15 +137,18 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return dataSetSchema.containsPartitionDimensions();
}
protected void removeDateIfExist(SemanticParseInfo semanticParseInfo) {
protected void removeDateIfExist(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
Set<String> removeFieldNames = new HashSet<>();
removeFieldNames.add(TimeDimensionEnum.DAY.getChName());
removeFieldNames.add(TimeDimensionEnum.WEEK.getChName());
removeFieldNames.add(TimeDimensionEnum.MONTH.getChName());
correctS2SQL = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames);
correctS2SQL = SqlRemoveHelper.removeSelect(correctS2SQL, removeFieldNames);
correctS2SQL = SqlRemoveHelper.removeGroupBy(correctS2SQL, removeFieldNames);
removeFieldNames.addAll(TimeDimensionEnum.getChNameList());
removeFieldNames.addAll(TimeDimensionEnum.getNameList());
Map<String, String> fieldNameMap = getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId());
removeFieldNames.removeIf(fieldName -> fieldNameMap.containsKey(fieldName));
if (!CollectionUtils.isEmpty(removeFieldNames)) {
correctS2SQL = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames);
correctS2SQL = SqlRemoveHelper.removeSelect(correctS2SQL, removeFieldNames);
correctS2SQL = SqlRemoveHelper.removeGroupBy(correctS2SQL, removeFieldNames);
}
semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL);
}
}

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java
View File

@@ -53,7 +53,7 @@ public class SchemaCorrector extends BaseSemanticCorrector {
if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) {
return;
}
removeDateIfExist(semanticParseInfo);
removeDateIfExist(chatQueryContext, semanticParseInfo);
}
private void correctAggFunction(SemanticParseInfo semanticParseInfo) {

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java
View File

@@ -31,7 +31,7 @@ public class TimeCorrector extends BaseSemanticCorrector {
if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) {
addDateIfNotExist(chatQueryContext, semanticParseInfo);
} else {
removeDateIfExist(semanticParseInfo);
removeDateIfExist(chatQueryContext, semanticParseInfo);
}
addLowerBoundDate(semanticParseInfo);
}