topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Remove the platform from the Docker Compose file and ensure the existence of a field with the same name before deleting a specific data date. (#1609)

Browse Source
This commit is contained in:
lexluo09
2024-08-28 15:22:54 +08:00
committed by GitHub
parent 0fc4713c4f
commit c4284f9784
4 changed files with 33 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docker/docker-compose.yml
View File

@@ -1,7 +1,6 @@
services: services:
chroma: chroma:
image: chromadb/chroma:0.5.3 image: chromadb/chroma:0.5.3
platform: linux/amd64
privileged: true privileged: true
container_name: supersonic_chroma container_name: supersonic_chroma
ports: ports:
@@ -23,7 +22,6 @@ services:
mysql: mysql:
image: mysql:8.0 image: mysql:8.0
privileged: true privileged: true
platform: linux/amd64
container_name: supersonic_mysql container_name: supersonic_mysql
environment: environment:
LANG: 'C.UTF-8' # 设置环境变量 LANG: 'C.UTF-8' # 设置环境变量
@@ -51,9 +49,8 @@ services:
retries: 5 retries: 5
db_init: db_init:
image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-0.9.6} image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-latest}
privileged: true privileged: true
platform: linux/amd64
container_name: supersonic_db_init container_name: supersonic_db_init
depends_on: depends_on:
mysql: mysql:
@@ -64,8 +61,8 @@ services:
sh -c " sh -c "
sleep 15 && sleep 15 &&
if ! mysql -h supersonic_mysql -usupersonic_user -psupersonic_password -e 'use supersonic_db; show tables;' | grep -q 's2_database'; then if ! mysql -h supersonic_mysql -usupersonic_user -psupersonic_password -e 'use supersonic_db; show tables;' | grep -q 's2_database'; then
mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-0.9.6}/conf/db/schema-mysql.sql && mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql.sql &&
mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-0.9.6}/conf/db/data-mysql.sql mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql.sql
else else
echo 'Database already initialized.' echo 'Database already initialized.'
fi fi
@@ -76,9 +73,8 @@ services:
- 8.8.4.4 - 8.8.4.4
supersonic_standalone: supersonic_standalone:
image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-0.9.6} image: supersonicbi/supersonic:${SUPERSONIC_VERSION:-latest}
privileged: true privileged: true
platform: linux/amd64
container_name: supersonic_standalone container_name: supersonic_standalone
environment: environment:
DB_HOST: supersonic_mysql DB_HOST: supersonic_mysql
@@ -103,9 +99,9 @@ services:
- 8.8.4.4 - 8.8.4.4
volumes: volumes:
#1.Named Volumes are best for persistent data managed by Docker. #1.Named Volumes are best for persistent data managed by Docker.
- supersonic_data:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-0.9.6} - supersonic_data:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}
#2.Bind Mounts are suitable for frequent modifications and debugging. #2.Bind Mounts are suitable for frequent modifications and debugging.
# - ./conf/langchain4j-prd.yaml:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/langchain4j-prd.yaml # - ./conf/application-prd.yaml:/usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/application-prd.yaml
#3.Detailed Bind Mounts offer more control over the mount behavior. #3.Detailed Bind Mounts offer more control over the mount behavior.
# - type: bind # - type: bind
# source: ./supersonic-standalone-${SUPERSONIC_VERSION:-latest}/logs # source: ./supersonic-standalone-${SUPERSONIC_VERSION:-latest}/logs

41
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java
View File

@@ -45,6 +45,20 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
protected Map<String, String> getFieldNameMap(ChatQueryContext chatQueryContext, Long dataSetId) { protected Map<String, String> getFieldNameMap(ChatQueryContext chatQueryContext, Long dataSetId) {
Map<String, String> result = getFieldNameMapFromDB(chatQueryContext, dataSetId);
if (chatQueryContext.containsPartitionDimensions(dataSetId)) {
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
}
return result;
}
private static Map<String, String> getFieldNameMapFromDB(ChatQueryContext chatQueryContext, Long dataSetId) {
SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema();
List<SchemaElement> dbAllFields = new ArrayList<>(); List<SchemaElement> dbAllFields = new ArrayList<>();
@@ -52,7 +66,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
dbAllFields.addAll(semanticSchema.getDimensions()); dbAllFields.addAll(semanticSchema.getDimensions());
// support fieldName and field alias // support fieldName and field alias
Map<String, String> result = dbAllFields.stream() return dbAllFields.stream()
.filter(entry -> dataSetId.equals(entry.getDataSetId())) .filter(entry -> dataSetId.equals(entry.getDataSetId()))
.flatMap(schemaElement -> { .flatMap(schemaElement -> {
Set<String> elements = new HashSet<>(); Set<String> elements = new HashSet<>();
@@ -63,14 +77,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return elements.stream(); return elements.stream();
}) })
.collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1)); .collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1));
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
return result;
} }
protected void addAggregateToMetric(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { protected void addAggregateToMetric(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
@@ -131,15 +137,18 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return dataSetSchema.containsPartitionDimensions(); return dataSetSchema.containsPartitionDimensions();
} }
protected void removeDateIfExist(SemanticParseInfo semanticParseInfo) { protected void removeDateIfExist(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
Set<String> removeFieldNames = new HashSet<>(); Set<String> removeFieldNames = new HashSet<>();
removeFieldNames.add(TimeDimensionEnum.DAY.getChName()); removeFieldNames.addAll(TimeDimensionEnum.getChNameList());
removeFieldNames.add(TimeDimensionEnum.WEEK.getChName()); removeFieldNames.addAll(TimeDimensionEnum.getNameList());
removeFieldNames.add(TimeDimensionEnum.MONTH.getChName()); Map<String, String> fieldNameMap = getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId());
correctS2SQL = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames); removeFieldNames.removeIf(fieldName -> fieldNameMap.containsKey(fieldName));
correctS2SQL = SqlRemoveHelper.removeSelect(correctS2SQL, removeFieldNames); if (!CollectionUtils.isEmpty(removeFieldNames)) {
correctS2SQL = SqlRemoveHelper.removeGroupBy(correctS2SQL, removeFieldNames); correctS2SQL = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames);
correctS2SQL = SqlRemoveHelper.removeSelect(correctS2SQL, removeFieldNames);
correctS2SQL = SqlRemoveHelper.removeGroupBy(correctS2SQL, removeFieldNames);
}
semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL); semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL);
} }
} }

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java
View File

@@ -53,7 +53,7 @@ public class SchemaCorrector extends BaseSemanticCorrector {
if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) { if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) {
return; return;
} }
removeDateIfExist(semanticParseInfo); removeDateIfExist(chatQueryContext, semanticParseInfo);
} }
private void correctAggFunction(SemanticParseInfo semanticParseInfo) { private void correctAggFunction(SemanticParseInfo semanticParseInfo) {

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java
View File

@@ -31,7 +31,7 @@ public class TimeCorrector extends BaseSemanticCorrector {
if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) { if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) {
addDateIfNotExist(chatQueryContext, semanticParseInfo); addDateIfNotExist(chatQueryContext, semanticParseInfo);
} else { } else {
removeDateIfExist(semanticParseInfo); removeDateIfExist(chatQueryContext, semanticParseInfo);
} }
addLowerBoundDate(semanticParseInfo); addLowerBoundDate(semanticParseInfo);
} }