.NET 8 compat test adjustments: 1) do not trim SDK, 2) support pattern to match output, 3) modify output truncation length (#3427)

src/Runner.Common/Constants.cs

@@ -281,6 +281,8 @@ namespace GitHub.Runner.Common
                 public static readonly string JobRequestType = "system.jobRequestType";
                 public static readonly string OrchestrationId = "system.orchestrationId";
                 public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility";
+                public static readonly string DotNet8CompatibilityOutputLength = "system.dotNet8CompatibilityOutputLength";
+                public static readonly string DotNet8CompatibilityOutputPattern = "system.dotNet8CompatibilityOutputPattern";
                 public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning";
             }
         }

src/Runner.Worker/OSWarningChecker.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
@@ -17,6 +18,8 @@ namespace GitHub.Runner.Worker
 
     public sealed class OSWarningChecker : RunnerService, IOSWarningChecker
     {
+        private static TimeSpan s_regexTimeout = TimeSpan.FromSeconds(1);
+
         public async Task CheckOSAsync(IExecutionContext context)
         {
             ArgUtil.NotNull(context, nameof(context));
@@ -68,13 +71,23 @@ namespace GitHub.Runner.Worker
                         var outputStr = string.Join("\n", output).Trim();
                         if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal))
                         {
+                            var pattern = context.Global.Variables.System_DotNet8CompatibilityOutputPattern;
+                            if (!string.IsNullOrEmpty(pattern))
+                            {
+                                var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, s_regexTimeout);
+                                if (!regex.IsMatch(outputStr))
+                                {
+                                    return;
+                                }
+                            }
+
                             var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning;
                             if (!string.IsNullOrEmpty(warningMessage))
                             {
                                 context.Warning(warningMessage);
                             }
 
-                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(output)}" });
+                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(context, output)}" });
                         }
                     }
                 }
@@ -83,14 +96,15 @@ namespace GitHub.Runner.Worker
             {
                 Trace.Error("An error occurred while testing .NET 8 compatibility'");
                 Trace.Error(ex);
-                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(output)}'" });
+                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(context, output)}'" });
             }
         }
 
-        private static string GetShortOutput(List<string> output)
+        private static string GetShortOutput(IExecutionContext context, List<string> output)
         {
+            var length = context.Global.Variables.System_DotNet8CompatibilityOutputLength ?? 200;
             var outputStr = string.Join("\n", output).Trim();
-            return outputStr.Length > 200 ? string.Concat(outputStr.Substring(0, 200), "[...]") : outputStr;
+            return outputStr.Length > length ? string.Concat(outputStr.Substring(0, length), "[...]") : outputStr;
         }
     }
 }

src/Runner.Worker/Variables.cs

@@ -74,6 +74,10 @@ namespace GitHub.Runner.Worker
 
         public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning);
 
+        public string System_DotNet8CompatibilityOutputPattern => Get(Constants.Variables.System.DotNet8CompatibilityOutputPattern);
+
+        public int? System_DotNet8CompatibilityOutputLength => GetInt(Constants.Variables.System.DotNet8CompatibilityOutputLength);
+
         public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName);
 
         public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false;

src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj

@@ -5,7 +5,6 @@
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
     <SelfContained>true</SelfContained>
-    <PublishTrimmed>true</PublishTrimmed>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>