[ubuntu] fix sporadic build fails (#2091)

* fix apt-get update

* Improve diagnostic

* handle "IPC connect call failed"

* Handle not-existing tool

* final refactoring

* remove todo

* remove which

images/linux/scripts/base/apt-mock-remove.sh

@@ -2,6 +2,6 @@
 
 prefix=/usr/local/bin
 
-for tool in apt apt-get apt-fast;do
+for tool in apt apt-get apt-fast apt-key;do
   sudo rm -f $prefix/$tool
 done

images/linux/scripts/base/apt-mock.sh

@@ -4,8 +4,8 @@
 
 prefix=/usr/local/bin
 
-for tool in apt apt-get apt-fast;do
+for real_tool in /usr/bin/apt /usr/bin/apt-get /usr/bin/apt-fast /usr/bin/apt-key;do
-  real_tool=`which $tool`
+  tool=`basename $real_tool`
   cat >$prefix/$tool <<EOT
 #!/bin/sh
 
@@ -14,15 +14,29 @@ while [ \$i -le 30 ];do
   err=\$(mktemp)
   $real_tool "\$@" 2>\$err
   result=\$?
-  if [ \$result -eq  0 ];then
-    break
-  fi
-  grep -q 'Could not get lock' \$err
-  held=\$?
-  if [ \$held -ne  0 ];then
-    break
-  fi
   cat \$err >&2
+
+  # no errors, continue
+  test \$result -eq 0 && break
+
+  retry=false
+
+  if grep -q 'Could not get lock' \$err;then
+    # apt db locked needs retry
+    retry=true
+  elif grep -q 'Could not open file /var/lib/apt/lists' \$err;then
+    # apt update is not completed, needs retry
+    retry=true
+  elif grep -q 'IPC connect call failed' \$err;then
+    # the delay should help with gpg-agent not ready
+    retry=true
+  fi
+
+  rm \$err
+  if [ \$retry = false ]; then
+    break
+  fi
+
   sleep 5
   echo "...retry \$i"
   i=\$((i + 1))

images/linux/ubuntu1604.json

@@ -61,6 +61,11 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "script": "{{template_dir}}/scripts/base/apt-mock.sh",
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts": [
@@ -76,11 +81,6 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
-        {
-            "type": "shell",
-            "script": "{{template_dir}}/scripts/base/apt-mock.sh",
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
         {
             "type": "shell",
             "script": "{{template_dir}}/scripts/base/limits.sh",

images/linux/ubuntu1804.json

@@ -61,6 +61,11 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "script": "{{template_dir}}/scripts/base/apt-mock.sh",
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts": [
@@ -79,11 +84,6 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
-        {
-            "type": "shell",
-            "script": "{{template_dir}}/scripts/base/apt-mock.sh",
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
         {
             "type": "shell",
             "script": "{{template_dir}}/scripts/base/limits.sh",

images/linux/ubuntu2004.json

@@ -63,6 +63,11 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "script": "{{template_dir}}/scripts/base/apt-mock.sh",
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts": [
@@ -89,11 +94,6 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
-        {
-            "type": "shell",
-            "script": "{{template_dir}}/scripts/base/apt-mock.sh",
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
         {
             "type": "shell",
             "script": "{{template_dir}}/scripts/base/limits.sh",