Handle keeping previous OSX versions more smoothly on Mac (#1381)

* Handle macOS upgrade smoothly

* cleanup

* misc cleanup

* final updates

* Update src/Misc/layoutbin/update.sh.template

Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com>

* Update src/Misc/layoutbin/update.sh.template

Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com>

* Upload telemetry and default to old method as needed

* minor fix

* add one more bit of logging

* some more telemetry

* quote variables to handle spaces

* tiny fix for ubuntu

* remove version and move telemetry to diag

* use full path

Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com>

src/Misc/layoutbin/update.sh.template

@@ -18,6 +18,8 @@ downloadrunnerversion=_DOWNLOAD_RUNNER_VERSION_
 logfile="_UPDATE_LOG_"
 restartinteractiverunner=_RESTART_INTERACTIVE_RUNNER_
 
+telemetryfile="$rootfolder/_diag/.telemetry"
+
 # log user who run the script
 date "+[%F %T-%4N] --------whoami--------" >> "$logfile" 2>&1
 whoami >> "$logfile" 2>&1
@@ -118,40 +120,101 @@ then
     exit 1
 fi
 
-# fix upgrade issue with macOS
+# fix upgrade issue with macOS when running as a service
+attemptedtargetedfix=0
 currentplatform=$(uname | awk '{print tolower($0)}')
-if [[ "$currentplatform" == 'darwin' ]]; then
+if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
-    # need a short-term fix for https://github.com/actions/runner/issues/743
+    # We needed a fix for https://github.com/actions/runner/issues/743
-    # we will recreate all the ./externals/node12/bin/node of the past 5 versions
+    # We will recreate the ./externals/node12/bin/node of the past runner version that launched the runnerlistener service
-    # v2.280.3 v2.280.2 v2.280.1 v2.279.0 v2.278.0
+    # Otherwise mac gatekeeper kills the processes we spawn on creation as we are running a process with no backing file
-    if [[ ! -e "$rootfolder/externals.2.280.3/node12/bin/node" ]]
+
+    # We need the pid for the nodejs loop, get that here, its the parent of the runner C# pid
+    # assumption here is only one process is invoking rootfolder/runsvc.sh
+    procgroup=$(ps x -o pgid,command | grep "$rootfolder/runsvc.sh" | grep -v grep | awk '{print $1}')
+    if [[ $? -eq 0 && -n "$procgroup" ]]
     then
-        mkdir -p "$rootfolder/externals.2.280.3/node12/bin"
+        # inspect the open file handles to find the node process
-        cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.280.3/node12/bin/node"
+        # we can't actually inspect the process using ps because it uses relative paths and doesn't follow symlinks
+        path=$(lsof -a -g "$procgroup" -F n | grep node12/bin/node | grep externals | tail -1 | cut -c2-)
+        if [[ $? -eq 0 && -n "$path" ]]
+        then
+            # trim the last 5 characters of the path '/node'
+            trimmedpath=$(dirname "$path")
+            if [[ $? -eq 0 && -n "$trimmedpath" ]]
+            then
+                attemptedtargetedfix=1
+                # Create the path if it does not exist
+                if [[ ! -e "$path" ]]
+                then
+                    date "+[%F %T-%4N] Creating fallback node at path $path" >> "$logfile" 2>&1
+                    mkdir -p "$trimmedpath"
+                    cp "$rootfolder/externals/node12/bin/node" "$path"
+                else
+                    date "+[%F %T-%4N] Path for fallback node exists, skipping creating $path" >> "$logfile" 2>&1
+                fi
+            else
+                date "+[%F %T-%4N] DarwinRunnerUpgrade: Failed to trim runner path. TrimmedPath: $trimmedpath, path: $path, pgid: $procgroup, root: $rootfolder" >> "$logfile" 2>&1
+                date "+[%F %T-%4N] DarwinRunnerUpgrade: Failed to trim runner path. TrimmedPath: $trimmedpath, path: $path, pgid: $procgroup, root: $rootfolder" >> "$telemetryfile" 2>&1
+            fi
+        else
+            date "+[%F %T-%4N] DarwinRunnerUpgrade: Failed to find runner path. Path: $path, pgid: $procgroup, root: $rootfolder" >> "$logfile" 2>&1
+            date "+[%F %T-%4N] DarwinRunnerUpgrade: Failed to find runner path. Path: $path, pgid: $procgroup, root: $rootfolder" >> "$telemetryfile" 2>&1
+        fi
+    else
+            date "+[%F %T-%4N] DarwinRunnerUpgrade: Failed to find runner pgid. pgid: $procgroup, root: $rootfolder" >> "$logfile" 2>&1
+            date "+[%F %T-%4N] DarwinRunnerUpgrade: Failed to find runner pgid. pgid: $procgroup, root: $rootfolder" >> "$telemetryfile" 2>&1
     fi
 
-    if [[ ! -e "$rootfolder/externals.2.280.2/node12/bin/node" ]]
+    if [ $attemptedtargetedfix -eq 0 ]
     then
-        mkdir -p "$rootfolder/externals.2.280.2/node12/bin"
-        cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.280.2/node12/bin/node"
-    fi
 
-    if [[ ! -e "$rootfolder/externals.2.280.1/node12/bin/node" ]]
+        date "+[%F %T-%4N] DarwinRunnerUpgrade: Defaulting to old macOS service fix" >> "$logfile" 2>&1
-    then
+        date "+[%F %T-%4N] DarwinRunnerUpgrade: Defaulting to old macOS service fix" >> "$telemetryfile" 2>&1
-        mkdir -p "$rootfolder/externals.2.280.1/node12/bin"
+        if [[ ! -e "$rootfolder/externals.2.280.3/node12/bin/node" ]]
-        cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.280.1/node12/bin/node"
+        then
-    fi
+            mkdir -p "$rootfolder/externals.2.280.3/node12/bin"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.280.3/node12/bin/node"
+        fi
 
-    if [[ ! -e "$rootfolder/externals.2.279.0/node12/bin/node" ]]
+        if [[ ! -e "$rootfolder/externals.2.280.2/node12/bin/node" ]]
-    then
+        then
-        mkdir -p "$rootfolder/externals.2.279.0/node12/bin"
+            mkdir -p "$rootfolder/externals.2.280.2/node12/bin"
-        cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.279.0/node12/bin/node"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.280.2/node12/bin/node"
-    fi
+        fi
 
-    if [[ ! -e "$rootfolder/externals.2.278.0/node12/bin/node" ]]
+        if [[ ! -e "$rootfolder/externals.2.280.1/node12/bin/node" ]]
-    then
+        then
-        mkdir -p "$rootfolder/externals.2.278.0/node12/bin"
+            mkdir -p "$rootfolder/externals.2.280.1/node12/bin"
-        cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.278.0/node12/bin/node"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.280.1/node12/bin/node"
+        fi
+
+        # GHES 3.2
+        if [[ ! -e "$rootfolder/externals.2.279.0/node12/bin/node" ]]
+        then
+            mkdir -p "$rootfolder/externals.2.279.0/node12/bin"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.279.0/node12/bin/node"
+        fi
+
+        # GHES 3.1.2 or later
+        if [[ ! -e "$rootfolder/externals.2.278.0/node12/bin/node" ]]
+        then
+            mkdir -p "$rootfolder/externals.2.278.0/node12/bin"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.278.0/node12/bin/node"
+        fi
+
+        # GHES 3.1.0
+        if [[ ! -e "$rootfolder/externals.2.276.1/node12/bin/node" ]]
+        then
+            mkdir -p "$rootfolder/externals.2.276.1/node12/bin"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.276.1/node12/bin/node"
+        fi
+
+        # GHES 3.0
+        if [[ ! -e "$rootfolder/externals.2.273.5/node12/bin/node" ]]
+        then
+            mkdir -p "$rootfolder/externals.2.273.5/node12/bin"
+            cp "$rootfolder/externals/node12/bin/node" "$rootfolder/externals.2.273.5/node12/bin/node"
+        fi
     fi
 fi
 

src/Runner.Common/Constants.cs

@@ -26,6 +26,7 @@ namespace GitHub.Runner.Common
         Certificates,
         Options,
         SetupInfo,
+        Telemetry
     }
 
     public static class Constants

src/Runner.Common/HostContext.cs

@@ -343,6 +343,12 @@ namespace GitHub.Runner.Common
                         ".setup_info");
                     break;
                 
+                case WellKnownConfigFile.Telemetry:
+                    path = Path.Combine(
+                        GetDirectory(WellKnownDirectory.Diag),
+                        ".telemetry");
+                    break;
+
                 default:
                     throw new NotSupportedException($"Unexpected well known config file: '{configFile}'");
             }

src/Runner.Worker/JobRunner.cs

@@ -7,6 +7,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Net.Http;
@@ -228,6 +229,9 @@ namespace GitHub.Runner.Worker
                 return result;
             }
 
+            // Load any upgrade telemetry
+            LoadFromTelemetryFile(jobContext.JobTelemetry);
+
             // Make sure we don't submit secrets as telemetry
             MaskTelemetrySecrets(jobContext.JobTelemetry);
 
@@ -285,6 +289,30 @@ namespace GitHub.Runner.Worker
             }
         }
 
+        private void LoadFromTelemetryFile(List<JobTelemetry> jobTelemetry)
+        {
+            try
+            {
+                var telemetryFilePath = HostContext.GetConfigFile(WellKnownConfigFile.Telemetry);
+                if (File.Exists(telemetryFilePath))
+                {
+                    var telemetryData = File.ReadAllText(telemetryFilePath, Encoding.UTF8);
+                    var telemetry = new JobTelemetry
+                    {
+                        Message = $"Runner File Telemetry:\n{telemetryData}",
+                        Type = JobTelemetryType.General
+                    };
+                    jobTelemetry.Add(telemetry);
+                    IOUtil.DeleteFile(telemetryFilePath);
+                }
+            }
+            catch (Exception e)
+            {
+                Trace.Error("Error when trying to load telemetry from telemetry file");
+                Trace.Error(e);
+            }
+        }
+
         private async Task ShutdownQueue(bool throwOnFailure)
         {
             if (_jobServerQueue != null)