Compare commits

..

5 Commits

Author SHA1 Message Date
Francesco Renzi
3b8cfdae4e Update releaseVersion to 2.298.0 2022-10-04 12:18:21 +00:00
Francesco Renzi
4935be5526 Prepare release notes for v2.298.0 (#2169) 2022-10-04 12:09:58 +00:00
Francesco Renzi
920fba93dc Add warning for users using deprecated commands (#2164) 2022-10-04 12:14:22 +01:00
JoannaaKL
949269104d Avastancu/joannaakl/service container error log (#2110)
* adding support for a service container docker logs

* Adding Unit test to ContainerOperationProvider

* Adding another test to ContainerOperationProvider

* placed the docker logs output in dedicated ##group section

* Removed the exception thrown if the service container was not healthy

* Removed duplicated logging to the executionContext

* Updated the container logs sub-section message

* Print service containers only if they were healthy
Unhealthy service logs are printed in ContainerHealthCheckLogs called prior to this step.

* Removed recently added method to inspect docker logs
The method was doing the same thing as the existing DockerLogs method.

* Added execution context error
This will make a failed health check more visible in the UI without disrupting the execution of the program.

* Removing the section 'Waiting for all services to be ready'

Since nested subsections are not being displayed properly and we already need one subsection per service error.

* Update src/Runner.Worker/Container/DockerCommandManager.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Update src/Test/L0/TestHostContext.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Change the logic for printing Service Containers logs

Service container logs will be printed in the 'Start containers' section only if there is an error.
Healthy services will have their logs printed in the 'Stop Containers' section.

* Removed unused import

* Added back section group.

* Moved service containers error logs to separate group sections

* Removed the test testing the old logic flow.

* Remove unnecessary 'IsAnyUnhealthy' flag

* Remove printHello() function

* Add newline to TestHostContext

* Remove unnecessary field 'UnhealthyContainers'

* Rename boolean flag indicating service container failure

* Refactor healthcheck logic to separate method to enable unit testing.

* Remove the default value for bool variable

* Update src/Runner.Worker/ContainerOperationProvider.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Update src/Runner.Worker/ContainerOperationProvider.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Rename Healthcheck back to ContainerHealthcheck

* Make test sequential

* Unextract the container error logs method

* remove test asserting thrown exception

* Add configure await

* Update src/Test/L0/Worker/ContainerOperationProviderL0.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Update src/Test/L0/Worker/ContainerOperationProviderL0.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Update src/Test/L0/Worker/ContainerOperationProviderL0.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Update src/Test/L0/Worker/ContainerOperationProviderL0.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Update src/Test/L0/Worker/ContainerOperationProviderL0.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Add back test asserting exception

* Check service exit code if there is no healtcheck configured

* Remove unnecessary healthcheck for healthy service container

* Revert "Check service exit code if there is no healtcheck configured"

This reverts commit fec24e8341.

Co-authored-by: Ava S <avastancu@github.com>
Co-authored-by: Tingluo Huang <tingluohuang@github.com>
2022-10-03 17:50:20 +02:00
Tauhid Anjum
dca4f67143 Adding a new vars context for non-secret variables (#2096)
* Adding a new vars context for non-secret variables

* Fix test case

* Trigger checks

* Remove variables from env context and environment varibale

* remove extra references

* Add prefix handling to configuration variables

* Fix test cases

* Consume variables using vars in context data

* removed action_yaml changes
2022-09-30 09:37:47 -04:00
13 changed files with 266 additions and 42 deletions

View File

@@ -1,13 +1,11 @@
## Features
- Created prerelease runner package for win-arm64 architecture (#2022)
- Added `GITHUB_STATE` and `GITHUB_OUTPUT` environment file commands (#2118)
- Service containers startup error logs are now included in workflow's logs (#2110)
## Bugs
- Fixed an issue where self hosted environments had their docker env's overwritten (#2107)
- Fixed an issue where step summaries for composite actions got overwritten (#2077)
<!-- ## Bugs -->
## Misc
- Bumped `actions/core` dependency (#2123)
- Added a feature flag to start warning on `save-state` and `set-output` deprecation (#2164)
- Prepare supporting `vars` in workflow templates (#2096)
## Windows x64
We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

View File

@@ -1 +1 @@
2.297.0
2.298.0

View File

@@ -159,6 +159,7 @@ namespace GitHub.Runner.Common
public static readonly string WorkerCrash = "WORKER_CRASH";
public static readonly string LowDiskSpace = "LOW_DISK_SPACE";
public static readonly string UnsupportedCommand = "UNSUPPORTED_COMMAND";
public static readonly string UnsupportedCommandMessage = "The `{0}` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/";
public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/";
public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";

View File

@@ -307,6 +307,17 @@ namespace GitHub.Runner.Worker
public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, ContainerInfo container)
{
if (context.Global.Variables.GetBoolean("DistributedTask.DeprecateStepOutputCommands") ?? false)
{
var issue = new Issue()
{
Type = IssueType.Warning,
Message = String.Format(Constants.Runner.UnsupportedCommandMessage, this.Command)
};
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.UnsupportedCommand;
context.AddIssue(issue);
}
if (!command.Properties.TryGetValue(SetOutputCommandProperties.Name, out string outputName) || string.IsNullOrEmpty(outputName))
{
throw new Exception("Required field 'name' is missing in ##[set-output] command.");
@@ -331,6 +342,17 @@ namespace GitHub.Runner.Worker
public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, ContainerInfo container)
{
if (context.Global.Variables.GetBoolean("DistributedTask.DeprecateStepOutputCommands") ?? false)
{
var issue = new Issue()
{
Type = IssueType.Warning,
Message = String.Format(Constants.Runner.UnsupportedCommandMessage, this.Command)
};
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.UnsupportedCommand;
context.AddIssue(issue);
}
if (!command.Properties.TryGetValue(SaveStateCommandProperties.Name, out string stateName) || string.IsNullOrEmpty(stateName))
{
throw new Exception("Required field 'name' is missing in ##[save-state] command.");
@@ -586,7 +608,7 @@ namespace GitHub.Runner.Worker
public void ProcessCommand(IExecutionContext context, string inputLine, ActionCommand command, ContainerInfo container)
{
ValidateLinesAndColumns(command, context);
command.Properties.TryGetValue(IssueCommandProperties.File, out string file);
command.Properties.TryGetValue(IssueCommandProperties.Line, out string line);
command.Properties.TryGetValue(IssueCommandProperties.Column, out string column);

View File

@@ -92,6 +92,8 @@ namespace GitHub.Runner.Worker.Container
public bool IsJobContainer { get; set; }
public bool IsAlpine { get; set; }
public bool FailedInitialization { get; set; }
public IDictionary<string, string> ContainerEnvironmentVariables
{
get

View File

@@ -98,12 +98,41 @@ namespace GitHub.Runner.Worker
await StartContainerAsync(executionContext, container);
}
await RunContainersHealthcheck(executionContext, containers);
}
public async Task RunContainersHealthcheck(IExecutionContext executionContext, List<ContainerInfo> containers)
{
executionContext.Output("##[group]Waiting for all services to be ready");
var unhealthyContainers = new List<ContainerInfo>();
foreach (var container in containers.Where(c => !c.IsJobContainer))
{
await ContainerHealthcheck(executionContext, container);
var healthcheck = await ContainerHealthcheck(executionContext, container);
if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase))
{
unhealthyContainers.Add(container);
}
else
{
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
}
}
executionContext.Output("##[endgroup]");
if (unhealthyContainers.Count > 0)
{
foreach (var container in unhealthyContainers)
{
executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed.");
await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId);
executionContext.Error($"Failed to initialize container {container.ContainerImage}");
container.FailedInitialization = true;
executionContext.Output("##[endgroup]");
}
throw new InvalidOperationException("One or more containers failed to start.");
}
}
public async Task StopContainersAsync(IExecutionContext executionContext, object data)
@@ -299,16 +328,15 @@ namespace GitHub.Runner.Worker
if (!string.IsNullOrEmpty(container.ContainerId))
{
if (!container.IsJobContainer)
if (!container.IsJobContainer && !container.FailedInitialization)
{
// Print logs for service container jobs (not the "action" job itself b/c that's already logged).
executionContext.Output($"Print service container logs: {container.ContainerDisplayName}");
executionContext.Output($"Print service container logs: {container.ContainerDisplayName}");
int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
}
executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}");
@@ -395,14 +423,14 @@ namespace GitHub.Runner.Worker
}
}
private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
private async Task<string> ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
{
string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\"";
string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault();
if (string.IsNullOrEmpty(serviceHealth))
{
// Container has no HEALTHCHECK
return;
return String.Empty;
}
var retryCount = 0;
while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase))
@@ -413,14 +441,7 @@ namespace GitHub.Runner.Worker
serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault();
retryCount++;
}
if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase))
{
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
}
else
{
throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}.");
}
return serviceHealth;
}
private async Task<string> ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container)

View File

@@ -73,6 +73,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String TimeoutMinutes = "timeout-minutes";
public const String Username = "username";
public const String Uses = "uses";
public const String Vars = "vars";
public const String VmImage = "vmImage";
public const String Volumes = "volumes";
public const String With = "with";

View File

@@ -631,6 +631,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.GitHub),
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.Needs),
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.Vars),
};
private static readonly INamedValueInfo[] s_stepNamedValues = new INamedValueInfo[]
{
@@ -643,6 +644,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.Runner),
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.Env),
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.Needs),
new NamedValueInfo<NoOperationNamedValue>(PipelineTemplateConstants.Vars),
};
private static readonly IFunctionInfo[] s_stepConditionFunctions = new IFunctionInfo[]
{

View File

@@ -465,6 +465,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
PipelineTemplateConstants.Job,
PipelineTemplateConstants.Runner,
PipelineTemplateConstants.Env,
PipelineTemplateConstants.Vars,
};
private readonly String[] s_expressionFunctionNames = new[]
{

View File

@@ -36,7 +36,8 @@
"workflow-env": {
"context": [
"github",
"secrets"
"secrets",
"vars"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -86,6 +87,7 @@
"context": [
"github",
"needs",
"vars",
"always(0,0)",
"failure(0,MAX)",
"cancelled(0,0)",
@@ -98,6 +100,7 @@
"context": [
"github",
"needs",
"vars",
"always(0,0)",
"failure(0,MAX)",
"cancelled(0,0)",
@@ -116,7 +119,8 @@
"strategy": {
"context": [
"github",
"needs"
"needs",
"vars"
],
"mapping": {
"properties": {
@@ -156,7 +160,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"one-of": [
"non-empty-string",
@@ -182,7 +187,8 @@
"needs",
"strategy",
"matrix",
"secrets"
"secrets",
"vars"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -204,7 +210,8 @@
"strategy",
"matrix",
"needs",
"env"
"env",
"vars"
],
"mapping": {
"properties": {
@@ -281,6 +288,7 @@
"job",
"runner",
"env",
"vars",
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
@@ -299,6 +307,7 @@
"job",
"runner",
"env",
"vars",
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
@@ -326,6 +335,7 @@
"job",
"runner",
"env",
"vars",
"hashFiles(1,255)"
],
"mapping": {
@@ -345,6 +355,7 @@
"job",
"runner",
"env",
"vars",
"hashFiles(1,255)"
],
"mapping": {
@@ -358,7 +369,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"one-of": [
"string",
@@ -384,7 +396,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -397,7 +410,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"one-of": [
"non-empty-string",
@@ -409,7 +423,8 @@
"context": [
"secrets",
"env",
"github"
"github",
"vars"
],
"mapping": {
"properties": {
@@ -443,7 +458,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"boolean": {}
},
@@ -453,7 +469,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"number": {}
},
@@ -463,7 +480,8 @@
"github",
"needs",
"strategy",
"matrix"
"matrix",
"vars"
],
"string": {}
},
@@ -479,6 +497,7 @@
"job",
"runner",
"env",
"vars",
"hashFiles(1,255)"
],
"boolean": {}
@@ -495,6 +514,7 @@
"job",
"runner",
"env",
"vars",
"hashFiles(1,255)"
],
"number": {}
@@ -510,7 +530,8 @@
"steps",
"job",
"runner",
"env"
"env",
"vars"
],
"string": {}
},
@@ -524,7 +545,8 @@
"steps",
"job",
"runner",
"env"
"env",
"vars"
],
"string": {}
},
@@ -540,6 +562,7 @@
"job",
"runner",
"env",
"vars",
"hashFiles(1,255)"
],
"string": {}

View File

@@ -0,0 +1,108 @@
using GitHub.Runner.Worker;
using GitHub.Runner.Worker.Container;
using Xunit;
using Moq;
using GitHub.Runner.Worker.Container.ContainerHooks;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using GitHub.DistributedTask.WebApi;
using System;
namespace GitHub.Runner.Common.Tests.Worker
{
public sealed class ContainerOperationProviderL0
{
private TestHostContext _hc;
private Mock<IExecutionContext> _ec;
private Mock<IDockerCommandManager> _dockerManager;
private Mock<IContainerHookManager> _containerHookManager;
private ContainerOperationProvider containerOperationProvider;
private Mock<IJobServerQueue> serverQueue;
private Mock<IPagingLogger> pagingLogger;
private List<string> healthyDockerStatus = new List<string> { "healthy" };
private List<string> unhealthyDockerStatus = new List<string> { "unhealthy" };
private List<string> dockerLogs = new List<string> { "log1", "log2", "log3" };
List<ContainerInfo> containers = new List<ContainerInfo>();
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertFailedTask()
{
//Arrange
Setup();
_dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(unhealthyDockerStatus));
//Act
try
{
await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers);
}
catch (InvalidOperationException)
{
//Assert
Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown()
{
//Arrange
Setup();
_dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(unhealthyDockerStatus));
//Act and Assert
await Assert.ThrowsAsync<InvalidOperationException>(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers));
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask()
{
//Arrange
Setup();
_dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(healthyDockerStatus));
//Act
await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers);
//Assert
Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded);
}
private void Setup([CallerMemberName] string testName = "")
{
containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" });
_hc = new TestHostContext(this, testName);
_ec = new Mock<IExecutionContext>();
serverQueue = new Mock<IJobServerQueue>();
pagingLogger = new Mock<IPagingLogger>();
_dockerManager = new Mock<IDockerCommandManager>();
_containerHookManager = new Mock<IContainerHookManager>();
containerOperationProvider = new ContainerOperationProvider();
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
_hc.SetSingleton<IJobServerQueue>(serverQueue.Object);
_hc.SetSingleton<IPagingLogger>(pagingLogger.Object);
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
_hc.SetSingleton<IContainerHookManager>(_containerHookManager.Object);
_ec.Setup(x => x.Global).Returns(new GlobalContext());
containerOperationProvider.Initialize(_hc);
}
}
}

View File

@@ -844,6 +844,51 @@ namespace GitHub.Runner.Common.Tests.Worker
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public void ActionVariables_AddedToVarsContext()
{
using (TestHostContext hc = CreateTestContext())
{
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
Guid jobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
{
Alias = Pipelines.PipelineConstants.SelfAlias,
Id = "github",
Version = "sha1"
});
jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
var inputVarsContext = new DictionaryContextData();
inputVarsContext["VARIABLE_1"] = new StringContextData("value1");
inputVarsContext["VARIABLE_2"] = new StringContextData("value2");
jobRequest.ContextData["vars"] = inputVarsContext;
// Arrange: Setup the paging logger.
var pagingLogger1 = new Mock<IPagingLogger>();
var jobServerQueue = new Mock<IJobServerQueue>();
hc.EnqueueInstance(pagingLogger1.Object);
hc.SetSingleton(jobServerQueue.Object);
var jobContext = new Runner.Worker.ExecutionContext();
jobContext.Initialize(hc);
jobContext.InitializeJob(jobRequest, CancellationToken.None);
var expected = new DictionaryContextData();
expected["VARIABLE_1"] = new StringContextData("value1");
expected["VARIABLE_2"] = new StringContextData("value1");
Assert.True(ExpressionValuesAssertEqual(expected, jobContext.ExpressionValues["vars"] as DictionaryContextData));
}
}
private bool ExpressionValuesAssertEqual(DictionaryContextData expect, DictionaryContextData actual)
{
foreach (var key in expect.Keys.ToList())

View File

@@ -1 +1 @@
2.297.0
2.298.0