[fix][headless]Fix NPE issue.

assembly/bin/supersonic-build.sh

@@ -35,12 +35,17 @@ function buildWebapp {
   chmod +x $projectDir/webapp/start-fe-prod.sh
   cd $projectDir/webapp
   sh ./start-fe-prod.sh
-  cp -fr  ./supersonic-webapp.tar.gz ${buildDir}/
   # check build result
   if [ $? -ne 0 ]; then
       echo "Failed to build frontend webapp."
       exit 1
   fi
+  cp -fr  ./supersonic-webapp.tar.gz ${buildDir}/
+  # check build result
+  if [ $? -ne 0 ]; then
+      echo "Failed to get supersonic webapp package."
+      exit 1
+  fi
   echo "finished building supersonic webapp"
 }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DuckdbAdaptor.java

@@ -36,6 +36,9 @@ public class DuckdbAdaptor extends DefaultDbAdaptor {
 
     @Override
     public String rewriteSql(String sql) {
+        if (sql == null) {
+            return null;
+        }
         return sql.replaceAll("`", "");
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -61,15 +61,7 @@ import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
@@ -372,7 +364,9 @@ public class ModelServiceImpl implements ModelService {
         metaFilter.setModelIds(Lists.newArrayList(modelId));
         List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
         List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
-        if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(dimensionResps)) {
+        boolean validMetric = metricResps.stream().anyMatch(metricResp -> Objects.equals(metricResp.getStatus(), StatusEnum.ONLINE.getCode()));
+        boolean validDimension = dimensionResps.stream().anyMatch(dimensionResp -> Objects.equals(dimensionResp.getStatus(), StatusEnum.ONLINE.getCode()));
+        if (validMetric || validDimension) {
             throw new RuntimeException("存在基于该模型创建的指标和维度, 暂不能删除, 请确认");
         }
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java

@@ -19,7 +19,6 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
 import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
 
 @Slf4j
@@ -73,10 +72,6 @@ public class QueryUtils {
             Map<String, String> nameTypePair, Map<String, MetricResp> metricRespMap,
             Map<String, DimensionResp> dimensionRespMap) {
         String nameEn = getName(column.getBizName());
-        if (nameEn.contains(JOIN_UNDERLINE)) {
-            nameEn = nameEn.split(JOIN_UNDERLINE)[1];
-        }
-        // set name
         if (namePair.containsKey(nameEn)) {
             column.setName(namePair.get(nameEn));
         } else {

launchers/standalone/src/main/resources/db/opensearch/ai_sql_memory.json

@@ -0,0 +1,95 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 6,
+      "number_of_replicas": 2,
+      "knn": true
+    },
+    "analysis": {
+      "analyzer": {
+        "ik_max_word_lowercase_html_strip": {
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ],
+          "char_filter": [
+            "html_strip"
+          ],
+          "tokenizer": "ik_max_word"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "metadata": {
+        "type": "object",
+        "properties": {
+          "dbSchema": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "queryId": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "question": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "sideInfo": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "sql": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          }
+        }
+      },
+      "text": {
+        "analyzer": "ik_max_word_lowercase_html_strip",
+        "search_analyzer": "ik_smart",
+        "type": "text"
+      },
+      "vector": {
+        "dimension": 512,
+        "method": {
+          "engine": "lucene",
+          "space_type": "cosinesimil",
+          "name": "hnsw",
+          "parameters": {
+            "ef_construction": 100,
+            "m": 16
+          }
+        },
+        "type": "knn_vector"
+      }
+    }
+  }
+}

launchers/standalone/src/main/resources/db/opensearch/ai_sql_memory_template.json

@@ -0,0 +1,98 @@
+{
+  "index_patterns": ["ai_sql_memory_*"],
+  "template": {
+    "settings": {
+      "index": {
+        "number_of_shards": 6,
+        "number_of_replicas": 2,
+        "knn": true
+      },
+      "analysis": {
+        "analyzer": {
+          "ik_max_word_lowercase_html_strip": {
+            "filter": [
+              "lowercase",
+              "asciifolding"
+            ],
+            "char_filter": [
+              "html_strip"
+            ],
+            "tokenizer": "ik_max_word"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "metadata": {
+          "type": "object",
+          "properties": {
+            "dbSchema": {
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              },
+              "type": "text"
+            },
+            "queryId": {
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              },
+              "type": "text"
+            },
+            "question": {
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              },
+              "type": "text"
+            },
+            "sideInfo": {
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              },
+              "type": "text"
+            },
+            "sql": {
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              },
+              "type": "text"
+            }
+          }
+        },
+        "text": {
+          "analyzer": "ik_max_word_lowercase_html_strip",
+          "search_analyzer": "ik_smart",
+          "type": "text"
+        },
+        "vector": {
+          "dimension": 512,
+          "method": {
+            "engine": "lucene",
+            "space_type": "cosinesimil",
+            "name": "hnsw",
+            "parameters": {
+              "ef_construction": 100,
+              "m": 16
+            }
+          },
+          "type": "knn_vector"
+        }
+      }
+    }
+  }
+}

launchers/standalone/src/main/resources/db/opensearch/ai_sql_meta_collection.json

@@ -0,0 +1,131 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 6,
+      "number_of_replicas": 2,
+      "knn": true
+    },
+    "analysis": {
+      "analyzer": {
+        "ik_max_word_lowercase_html_strip": {
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ],
+          "char_filter": [
+            "html_strip"
+          ],
+          "tokenizer": "ik_max_word"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "metadata": {
+        "type": "object",
+        "properties": {
+          "bizName": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "defaultAgg": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "domainId": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "id": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "modelId": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "name": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "newName": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "queryId": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "type": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          }
+        }
+      },
+      "text": {
+        "analyzer": "ik_max_word_lowercase_html_strip",
+        "search_analyzer": "ik_smart",
+        "type": "text"
+      },
+      "vector": {
+        "dimension": 512,
+        "method": {
+          "engine": "lucene",
+          "space_type": "cosinesimil",
+          "name": "hnsw",
+          "parameters": {
+            "ef_construction": 100,
+            "m": 16
+          }
+        },
+        "type": "knn_vector"
+      }
+    }
+  }
+}

launchers/standalone/src/main/resources/db/opensearch/ai_sql_preset_query_collection.json

@@ -0,0 +1,59 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 6,
+      "number_of_replicas": 2,
+      "knn": true
+    },
+    "analysis": {
+      "analyzer": {
+        "ik_max_word_lowercase_html_strip": {
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ],
+          "char_filter": [
+            "html_strip"
+          ],
+          "tokenizer": "ik_max_word"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "metadata": {
+        "type": "object",
+        "properties": {
+          "queryId": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      },
+      "text": {
+        "type": "text",
+        "analyzer": "ik_max_word_lowercase_html_strip",
+        "search_analyzer": "ik_smart"
+      },
+      "vector": {
+        "type": "knn_vector",
+        "dimension": 512,
+        "method": {
+          "name": "hnsw",
+          "engine": "lucene",
+          "space_type": "cosinesimil",
+          "parameters": {
+            "ef_construction": 100,
+            "m": 16
+          }
+        }
+      }
+    }
+  }
+}

launchers/standalone/src/main/resources/db/opensearch/ai_sql_text2dsl_agent_collection.json

@@ -0,0 +1,95 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 6,
+      "number_of_replicas": 2,
+      "knn": true
+    },
+    "analysis": {
+      "analyzer": {
+        "ik_max_word_lowercase_html_strip": {
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ],
+          "char_filter": [
+            "html_strip"
+          ],
+          "tokenizer": "ik_max_word"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "metadata": {
+        "type": "object",
+        "properties": {
+          "dbSchema": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "queryId": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "question": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "sideInfo": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          },
+          "sql": {
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            },
+            "type": "text"
+          }
+        }
+      },
+      "text": {
+        "analyzer": "ik_max_word_lowercase_html_strip",
+        "search_analyzer": "ik_smart",
+        "type": "text"
+      },
+      "vector": {
+        "dimension": 512,
+        "method": {
+          "engine": "lucene",
+          "space_type": "cosinesimil",
+          "name": "hnsw",
+          "parameters": {
+            "ef_construction": 100,
+            "m": 16
+          }
+        },
+        "type": "knn_vector"
+      }
+    }
+  }
+}

launchers/standalone/src/main/resources/db/opensearch/opensearch_create_index.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# This script is used to create index pattern for AI SQL
+# Usage: ./opensearch_create_index.sh
+# Note: Please make sure the opensearch is running and the index pattern is not exist。
+# To confirm the vector dimension, the default is 512.
+# If you need to modify it, please first adjust the corresponding index configuration.
+# configure the following parameters for opensearch
+# es_host: the host of opensearch
+# es_user: the user of opensearch
+# es_password: the password of opensearch
+# es_index_prefix: the index prefix of opensearch
+es_host="https://opensearch-node:7799"
+es_user="admin"
+es_password="admin"
+es_index_prefix="ai_sql"
+
+echo "Creating index pattern for AI SQL"
+echo "creating index ${es_index_prefix}_meta_collection"
+curl -X PUT "${es_host}/${es_index_prefix}_meta_collection" \
+  -u "${es_user}:${es_password}" \
+  -H "Content-Type: application/json" \
+  -d @ai_sql_meta_collection.json
+
+echo "creating index ${es_index_prefix}_text2dsl_agent_collection"
+curl -X PUT "${es_host}/${es_index_prefix}_text2dsl_agent_collection" \
+  -u "${es_user}:${es_password}" \
+  -H "Content-Type: application/json" \
+  -d @ai_sql_text2dsl_agent_collection.json
+
+echo "creating index ${es_index_prefix}_preset_query_collection"
+curl -X PUT "${es_host}/${es_index_prefix}_preset_query_collection" \
+  -u "${es_user}:${es_password}" \
+  -H "Content-Type: application/json" \
+  -d @ai_sql_preset_query_collection.json
+
+for i in {1..10}; do
+  echo "creating index ${es_index_prefix}_memory_${i}"
+  curl -X PUT "${es_host}/${es_index_prefix}_memory_${i}" \
+    -u "${es_user}:${es_password}" \
+    -H "Content-Type: application/json" \
+    -d @ai_sql_memory.json
+done

webapp/packages/chat-sdk/src/common/type.ts

@@ -178,6 +178,7 @@ export type QueryDataType = {
 export type ColumnType = {
   authorized: boolean;
   name: string;
+  nameEn: string;
   bizName: string;
   showType: string;
   type: string;

webapp/start-fe-dev.sh

@@ -19,7 +19,10 @@ cd ./packages/chat-sdk
 pnpm i
 
 pnpm run build
-
+if [ $? -ne 0 ]; then
+    echo "Failed to build chat sdk."
+    exit 1
+fi
 pnpm link --global
 
 cd ../supersonic-fe

webapp/start-fe-prod.sh

@@ -23,6 +23,10 @@ cd ./packages/chat-sdk
 pnpm i
 
 pnpm run build
+if [ $? -ne 0 ]; then
+    echo "Failed to build chat sdk."
+    exit 1
+fi
 
 pnpm link --global
 
@@ -33,6 +37,10 @@ pnpm link ../chat-sdk
 pnpm i
 
 pnpm run build:os-local
+if [ $? -ne 0 ]; then
+    echo "Failed to build supersonic-fe."
+    exit 1
+fi
 
 tar -zcvf supersonic-webapp.tar.gz ./supersonic-webapp