diff --git a/__tests__/fixtures/not_supported.yml b/__tests__/fixtures/not_supported.yml
index fac330ad..8af7523a 100644
--- a/__tests__/fixtures/not_supported.yml
+++ b/__tests__/fixtures/not_supported.yml
@@ -1,2 +1,3 @@
 label:
-  - unknown: 'this-is-not-supported'
+  - all:
+      - unknown: 'this-is-not-supported'
diff --git a/dist/index.js b/dist/index.js
index 38d5b586..14db90c4 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -467,7 +467,9 @@ exports.checkAny = checkAny;
 // equivalent to "Array.every()" but expanded for debugging and clarity
 function checkAll(matchConfigs, changedFiles) {
     core.debug(`  checking "all" patterns`);
-    if (!matchConfigs.length) {
+    if (!matchConfigs.length ||
+        // Make sure that all the configs have keys that we can check for
+        !matchConfigs.some(configOption => ALLOWED_CONFIG_KEYS.includes(Object.keys(configOption)[0]))) {
         core.debug(`  no "all" patterns to check`);
         return false;
     }
diff --git a/src/labeler.ts b/src/labeler.ts
index d42b03e1..72bce200 100644
--- a/src/labeler.ts
+++ b/src/labeler.ts
@@ -260,7 +260,13 @@ export function checkAll(
   changedFiles: string[]
 ): boolean {
   core.debug(`  checking "all" patterns`);
-  if (!matchConfigs.length) {
+  if (
+    !matchConfigs.length ||
+    // Make sure that all the configs have keys that we can check for
+    !matchConfigs.some(configOption =>
+      ALLOWED_CONFIG_KEYS.includes(Object.keys(configOption)[0])
+    )
+  ) {
     core.debug(`  no "all" patterns to check`);
     return false;
   }