WIP: generate coverage report

Todo: 1. Use newest hash 2. Use index.html stats

images/Dockerfile

@@ -2,7 +2,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build
 
 ARG RUNNER_VERSION
 ARG RUNNER_ARCH="x64"
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.3.2
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.3.1
 ARG DOCKER_VERSION=20.10.23
 
 RUN apt update -y && apt install curl unzip -y

src/Runner.Common/Constants.cs

@@ -155,7 +155,6 @@ namespace GitHub.Runner.Common
             {
                 public static readonly string DiskSpaceWarning = "runner.diskspace.warning";
                 public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action";
-                public static readonly string LogTemplateErrorsToTraceWriter = "DistributedTask.LogTemplateErrorsToTraceWriter";
                 public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate";
                 public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks";
             }

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -137,7 +137,7 @@ namespace GitHub.Runner.Listener.Configuration
                     GitHubAuthResult authResult = await GetTenantCredential(inputUrl, registerToken, Constants.RunnerEvent.Register);
                     runnerSettings.ServerUrl = authResult.TenantUrl;
                     runnerSettings.UseV2Flow = authResult.UseV2Flow;
-                    Trace.Info($"Using V2 flow: {runnerSettings.UseV2Flow}");
+                    _term.WriteLine($"Using V2 flow: {runnerSettings.UseV2Flow}");
                     creds = authResult.ToVssCredentials();
                     Trace.Info("cred retrieved via GitHub auth");
                 }

src/Runner.Worker/ActionManager.cs

@@ -317,28 +317,15 @@ namespace GitHub.Runner.Worker
 
             if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
             {
-                if (FeatureManager.IsContainerHooksEnabled(executionContext.Global.Variables))
+                Trace.Info("Load action that reference container from registry.");
+                CachedActionContainers.TryGetValue(action.Id, out var container);
+                ArgUtil.NotNull(container, nameof(container));
+                definition.Data.Execution = new ContainerActionExecutionData()
                 {
-                    Trace.Info("Load action that will run container through container hooks.");
-                    var containerAction = action.Reference as Pipelines.ContainerRegistryReference;
-                    definition.Data.Execution = new ContainerActionExecutionData()
-                    {
-                        Image = containerAction.Image,
-                    };
-                    Trace.Info($"Using action container image: {containerAction.Image}.");
-                }
-                else
-                {
-                    Trace.Info("Load action that reference container from registry.");
-                    CachedActionContainers.TryGetValue(action.Id, out var container);
-                    ArgUtil.NotNull(container, nameof(container));
-                    definition.Data.Execution = new ContainerActionExecutionData()
-                    {
-                        Image = container.ContainerImage
-                    };
+                    Image = container.ContainerImage
+                };
 
-                    Trace.Info($"Using action container image: {container.ContainerImage}.");
-                }
+                Trace.Info($"Using action container image: {container.ContainerImage}.");
             }
             else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
             {

src/Runner.Worker/ActionManifestManager.cs

@@ -52,7 +52,7 @@ namespace GitHub.Runner.Worker
 
         public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile)
         {
-            var templateContext = CreateTemplateContext(executionContext, null, LogTemplateErrorsToTraceWriter(executionContext));
+            var templateContext = CreateTemplateContext(executionContext);
             ActionDefinitionData actionDefinition = new();
 
             // Clean up file name real quick
@@ -303,8 +303,7 @@ namespace GitHub.Runner.Worker
 
         private TemplateContext CreateTemplateContext(
             IExecutionContext executionContext,
-            IDictionary<string, PipelineContextData> extraExpressionValues = null,
-            bool addErrorsToTraceWriter = true)
+            IDictionary<string, PipelineContextData> extraExpressionValues = null)
         {
             var result = new TemplateContext
             {
@@ -316,7 +315,6 @@ namespace GitHub.Runner.Worker
                     maxBytes: 10 * 1024 * 1024),
                 Schema = _actionManifestSchema,
                 TraceWriter = executionContext.ToTemplateTraceWriter(),
-                LogErrorsToTraceWriter = addErrorsToTraceWriter
             };
 
             // Expression values from execution context
@@ -450,7 +448,7 @@ namespace GitHub.Runner.Worker
                         };
                     }
                 }
-                else if (string.Equals(usingToken.Value, "node12", StringComparison.OrdinalIgnoreCase) ||
+                else if (string.Equals(usingToken.Value, "node12", StringComparison.OrdinalIgnoreCase)||
                          string.Equals(usingToken.Value, "node16", StringComparison.OrdinalIgnoreCase))
                 {
                     if (string.IsNullOrEmpty(mainToken?.Value))
@@ -541,13 +539,6 @@ namespace GitHub.Runner.Worker
                 }
             }
         }
-
-        private bool LogTemplateErrorsToTraceWriter(IExecutionContext executionContext)
-        {
-            if (executionContext == null || executionContext.Global == null || executionContext.Global.EnvironmentVariables == null) return true;
-            executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Runner.Features.LogTemplateErrorsToTraceWriter, out var logErrorsAsDebug);
-            return StringUtil.ConvertToBoolean(logErrorsAsDebug, defaultValue: true);
-        }
     }
 }
 

src/Runner.Worker/Container/ContainerHooks/HookInput.cs

@@ -83,7 +83,7 @@ namespace GitHub.Runner.Worker.Container.ContainerHooks
         public HookContainer(ContainerInfo container)
         {
             Image = container.ContainerImage;
-            EntryPointArgs = container.ContainerEntryPointArgs?.Split(' ').Select(arg => arg.Trim()).Where(arg => !string.IsNullOrEmpty(arg)) ?? new List<string>();
+            EntryPointArgs = container.ContainerEntryPointArgs?.Split(' ').Select(arg => arg.Trim()) ?? new List<string>();
             EntryPoint = container.ContainerEntryPoint;
             WorkingDirectory = container.ContainerWorkDirectory;
             CreateOptions = container.ContainerCreateOptions;

src/Runner.Worker/ExecutionContext.cs

@@ -21,7 +21,6 @@ using Newtonsoft.Json;
 using Sdk.RSWebApi.Contracts;
 using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating;
 using Pipelines = GitHub.DistributedTask.Pipelines;
-using constants = GitHub.Runner.Common.Constants;
 
 namespace GitHub.Runner.Worker
 {
@@ -1426,5 +1425,4 @@ namespace GitHub.Runner.Worker
         public static readonly string Notice = "##[notice]";
         public static readonly string Debug = "##[debug]";
     }
-
 }

src/Runner.Worker/JobRunner.cs

@@ -150,11 +150,6 @@ namespace GitHub.Runner.Worker
                 _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
                 jobContext.SetRunnerContext("name", _runnerSettings.AgentName);
 
-                if (jobContext.Global.Variables.TryGetValue(WellKnownDistributedTaskVariables.RunnerEnvironment, out var runnerEnvironment))
-                {
-                    jobContext.SetRunnerContext("environment", runnerEnvironment);
-                }
-
                 string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
                 Directory.CreateDirectory(toolsDirectory);
                 jobContext.SetRunnerContext("tool_cache", toolsDirectory);

src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateContext.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
@@ -34,11 +34,6 @@ namespace GitHub.DistributedTask.ObjectTemplating
                 m_errors = value;
             }
         }
-        /// <summary>
-        /// Because TraceWriter has access to the ExecutionContext and is logging template errors, duplicated issues are reported.
-        /// By setting LogErrorsToTraceWriter = false TemplateContext will add errors only to TemplateValidationErrors 
-        /// </summary>
-        internal bool LogErrorsToTraceWriter = true;
 
         /// <summary>
         /// Available functions within expression contexts
@@ -125,9 +120,15 @@ namespace GitHub.DistributedTask.ObjectTemplating
             }
         }
 
+        internal void Error(TemplateValidationError error)
+        {
+            Errors.Add(error);
+            TraceWriter.Error(error.Message);
+        }
+
         internal void Error(
-           TemplateToken value,
-           Exception ex)
+            TemplateToken value,
+            Exception ex)
         {
             Error(value?.FileId, value?.Line, value?.Column, ex);
         }
@@ -140,11 +141,7 @@ namespace GitHub.DistributedTask.ObjectTemplating
         {
             var prefix = GetErrorPrefix(fileId, line, column);
             Errors.Add(prefix, ex);
-
-            if (LogErrorsToTraceWriter)
-            {
-                TraceWriter.Error(prefix, ex);
-            }
+            TraceWriter.Error(prefix, ex);
         }
 
         internal void Error(
@@ -161,13 +158,13 @@ namespace GitHub.DistributedTask.ObjectTemplating
             String message)
         {
             var prefix = GetErrorPrefix(fileId, line, column);
-            var fullMessage = !String.IsNullOrEmpty(prefix) ? $"{prefix} {message}" : message;
-
-            Errors.Add(fullMessage);
-            if (LogErrorsToTraceWriter)
+            if (!String.IsNullOrEmpty(prefix))
             {
-                TraceWriter.Error(fullMessage);
+                message = $"{prefix} {message}";
             }
+
+            Errors.Add(message);
+            TraceWriter.Error(message);
         }
 
         internal INamedValueInfo[] GetExpressionNamedValues()

src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 
 namespace GitHub.DistributedTask.WebApi
 {
@@ -6,6 +6,5 @@ namespace GitHub.DistributedTask.WebApi
     {
         public static readonly String JobId = "system.jobId";
         public static readonly String RunnerLowDiskspaceThreshold = "system.runner.lowdiskspacethreshold";
-        public static readonly String RunnerEnvironment = "system.runnerEnvironment";
     }
 }

src/Test/L0/Sdk/DTObjectTemplating/TemplateContextL0.cs

@@ -1,275 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using GitHub.DistributedTask.ObjectTemplating.Tokens;
-using Xunit;
-
-namespace GitHub.DistributedTask.ObjectTemplating.Tests
-{
-    public sealed class TemplateContextL0
-    {
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithTokenAndException()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("(Line: 1, Col: 1): Exception of type 'System.Exception' was thrown.") };
-
-            context.Error(new StringToken(1, 1, 1, "some-token"), new System.Exception());
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithNullTokenAndException()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("System.Exception: Exception of type 'System.Exception' was thrown.") };
-
-            context.Error(null, new System.Exception());
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithTokenAndMessage()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("(Line: 1, Col: 1): message") };
-
-            context.Error(new StringToken(1, 1, 1, "some-token"), "message");
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithNullTokenAndMessage()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("message") };
-
-            context.Error(null, "message");
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithException()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("(Line: 2, Col: 3): Fatal template exception") };
-            List<string> expectedTracewriterErrors = new List<string> { "(Line: 2, Col: 3):" };
-
-            context.Error(1, 2, 3, new Exception("Fatal template exception"));
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-            ListTraceWriter listTraceWriter = (ListTraceWriter)context.TraceWriter;
-            List<string> tracewriterErrors = listTraceWriter.GetErrors();
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-            Assert.True(expectedTracewriterErrors.SequenceEqual(tracewriterErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithExceptionAndInnerException()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("(Line: 2, Col: 3): Fatal template exception"),
-            new TemplateValidationError("(Line: 2, Col: 3): Inner Exception") };
-            List<string> expectedTracewriterErrors = new List<string> { "(Line: 2, Col: 3):" };
-
-            Exception e = new Exception("Fatal template exception", new Exception("Inner Exception"));
-
-            context.Error(1, 2, 3, e);
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-            ListTraceWriter listTraceWriter = (ListTraceWriter)context.TraceWriter;
-            List<string> tracewriterErrors = listTraceWriter.GetErrors();
-
-            Assert.Equal(2, templateValidationErrors.Count);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-            Assert.True(expectedTracewriterErrors.SequenceEqual(tracewriterErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithLineNumbersAndMessage()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("(Line: 2, Col: 3): Fatal template exception") };
-            List<string> expectedTracewriterErrors = new List<string> { "(Line: 2, Col: 3): Fatal template exception" };
-
-            context.Error(1, 2, 3, "Fatal template exception");
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-            ListTraceWriter listTraceWriter = (ListTraceWriter)context.TraceWriter;
-            List<string> tracewriterErrors = listTraceWriter.GetErrors();
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-            Assert.True(expectedTracewriterErrors.SequenceEqual(tracewriterErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithNullLineNumbersAndMessage()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("Fatal template exception") };
-            List<string> expectedTracewriterErrors = new List<string> { "Fatal template exception" };
-
-            context.Error(null, null, null, "Fatal template exception");
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-            ListTraceWriter listTraceWriter = (ListTraceWriter)context.TraceWriter;
-            List<string> tracewriterErrors = listTraceWriter.GetErrors();
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-            Assert.True(expectedTracewriterErrors.SequenceEqual(tracewriterErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithLineNumbersAndException()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("(Line: 2, Col: 3): Fatal template exception") };
-            List<string> expectedTracewriterErrors = new List<string> { "(Line: 2, Col: 3):" };
-
-            context.Error(1, 2, 3, new Exception("Fatal template exception"));
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-            ListTraceWriter listTraceWriter = (ListTraceWriter)context.TraceWriter;
-            List<string> tracewriterErrors = listTraceWriter.GetErrors();
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-            Assert.True(expectedTracewriterErrors.SequenceEqual(tracewriterErrors));
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void VerifyErrorWithNullLineNumbersAndException()
-        {
-            TemplateContext context = buildContext();
-
-            List<TemplateValidationError> expectedErrors = new List<TemplateValidationError> { new TemplateValidationError("System.Exception: Fatal template exception") };
-            List<string> expectedTracewriterErrors = new List<string> { "" };
-
-            context.Error(null, null, null, new Exception("Fatal template exception"));
-
-            List<TemplateValidationError> templateValidationErrors = toList(context.Errors.GetEnumerator());
-            ListTraceWriter listTraceWriter = (ListTraceWriter)context.TraceWriter;
-            List<string> tracewriterErrors = listTraceWriter.GetErrors();
-
-            Assert.Single(templateValidationErrors);
-            Assert.True(areEqual(expectedErrors, templateValidationErrors));
-            Assert.True(expectedTracewriterErrors.SequenceEqual(tracewriterErrors));
-        }
-
-        private TemplateContext buildContext()
-        {
-            return new TemplateContext
-            {
-                // CancellationToken = CancellationToken.None,
-                Errors = new TemplateValidationErrors(10, int.MaxValue), // Don't truncate error messages otherwise we might not scrub secrets correctly
-                Memory = new TemplateMemory(
-                maxDepth: 100,
-                maxEvents: 1000000,
-                maxBytes: 10 * 1024 * 1024),
-                Schema = null,
-                TraceWriter = new ListTraceWriter(),
-            };
-        }
-
-        private List<TemplateValidationError> toList(IEnumerator<TemplateValidationError> enumerator)
-        {
-            List<TemplateValidationError> result = new();
-            while (enumerator.MoveNext())
-            {
-                TemplateValidationError err = enumerator.Current;
-                result.Add(err);
-            }
-            return result;
-        }
-
-        private bool areEqual(List<TemplateValidationError> l1, List<TemplateValidationError> l2)
-        {
-            if (l1.Count != l2.Count) return false;
-
-            var twoLists = l1.Zip(l2, (l1Error, l2Error) => new { Elem1 = l1Error, Elem2 = l2Error });
-
-            foreach (var elem in twoLists)
-            {
-                if (elem.Elem1.Message != elem.Elem2.Message) return false;
-                if (elem.Elem1.Code != elem.Elem2.Code) return false;
-            }
-            return true;
-        }
-
-        internal sealed class ListTraceWriter : ITraceWriter
-        {
-
-            private List<string> errors = new();
-            private List<string> infoMessages = new();
-            private List<string> verboseMessages = new();
-            public void Error(string format, params object[] args)
-            {
-                errors.Add(string.Format(System.Globalization.CultureInfo.CurrentCulture, $"{format}", args));
-            }
-
-            public void Info(string format, params object[] args)
-            {
-                throw new NotImplementedException();
-            }
-
-            public void Verbose(string format, params object[] args)
-            {
-                throw new NotImplementedException();
-            }
-
-            public List<string> GetErrors()
-            {
-                return errors;
-            }
-        }
-    }
-}

src/Test/Test.csproj

@@ -15,6 +15,10 @@
     </ItemGroup>
 
     <ItemGroup>
+        <PackageReference Include="coverlet.collector" Version="3.2.0">
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+          <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
         <PackageReference Include="xunit" Version="2.4.1" />
         <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

src/dev.sh

@@ -194,6 +194,10 @@ function layout ()
 
 function runtest ()
 {
+    if [[ ! -d "$LAYOUT_DIR" ]]; then
+        echo "$LAYOUT_DIR doesn't exist. Generating it now ..."
+        layout
+    fi
     heading "Testing ..."
 
     if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then
@@ -203,6 +207,13 @@ function runtest ()
     dotnet msbuild -t:test -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed "failed tests"
 }
 
+function coverage ()
+{
+    heading "Coverage ..."
+    cd Test && dotnet test --collect:"XPlat Code Coverage;Format=json"
+    # reportgenerator -reports:"/workspaces/runner/src/Test/TestResults/ecf2bd75-83e9-489a-9339-d61293abf98b/coverage.cobertura.xml" -targetdir:"coveragereport" -reporttypes:Html
+}
+
 function format()
 {
     heading "Formatting..."
@@ -369,6 +380,7 @@ case $DEV_CMD in
     "p") package;;
     "format") format;;
     "f") format;;
+    "c") coverage;;
     *) echo "Invalid cmd.  Use build(b), test(t), layout(l), package(p), or format(f)";;
 esac