diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index 32e55eb3c..9c114939e 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -92,8 +92,6 @@ namespace GitHub.Runner.Worker.Container public bool IsJobContainer { get; set; } public bool IsAlpine { get; set; } - public bool FailedInitialization { get; set; } - public IDictionary ContainerEnvironmentVariables { get diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 2fc597308..73472795c 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -98,41 +98,12 @@ namespace GitHub.Runner.Worker await StartContainerAsync(executionContext, container); } - await RunContainersHealthcheck(executionContext, containers); - } - - public async Task RunContainersHealthcheck(IExecutionContext executionContext, List containers) - { executionContext.Output("##[group]Waiting for all services to be ready"); - - var unhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { - var healthcheck = await ContainerHealthcheck(executionContext, container); - - if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) - { - unhealthyContainers.Add(container); - } - else - { - executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); - } + await ContainerHealthcheck(executionContext, container); } executionContext.Output("##[endgroup]"); - - if (unhealthyContainers.Count > 0) - { - foreach (var container in unhealthyContainers) - { - executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); - await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); - executionContext.Error($"Failed to initialize container {container.ContainerImage}"); - container.FailedInitialization = true; - executionContext.Output("##[endgroup]"); - } - throw new InvalidOperationException("One or more containers failed to start."); - } } public async Task StopContainersAsync(IExecutionContext executionContext, object data) @@ -328,15 +299,16 @@ namespace GitHub.Runner.Worker if (!string.IsNullOrEmpty(container.ContainerId)) { - if (!container.IsJobContainer && !container.FailedInitialization) + if (!container.IsJobContainer) { - executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); + // Print logs for service container jobs (not the "action" job itself b/c that's already logged). + executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); - int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); - if (logsExitCode != 0) - { - executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); - } + int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); + if (logsExitCode != 0) + { + executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); + } } executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}"); @@ -423,14 +395,14 @@ namespace GitHub.Runner.Worker } } - private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) + private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) { // Container has no HEALTHCHECK - return String.Empty; + return; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) @@ -441,7 +413,14 @@ namespace GitHub.Runner.Worker serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); retryCount++; } - return serviceHealth; + if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) + { + executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); + } + else + { + throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); + } } private async Task ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs deleted file mode 100644 index bfc09f343..000000000 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ /dev/null @@ -1,108 +0,0 @@ -using GitHub.Runner.Worker; -using GitHub.Runner.Worker.Container; -using Xunit; -using Moq; -using GitHub.Runner.Worker.Container.ContainerHooks; -using System.Threading.Tasks; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using GitHub.DistributedTask.WebApi; -using System; - -namespace GitHub.Runner.Common.Tests.Worker -{ - - public sealed class ContainerOperationProviderL0 - { - - private TestHostContext _hc; - private Mock _ec; - private Mock _dockerManager; - private Mock _containerHookManager; - private ContainerOperationProvider containerOperationProvider; - private Mock serverQueue; - private Mock pagingLogger; - private List healthyDockerStatus = new List { "healthy" }; - private List unhealthyDockerStatus = new List { "unhealthy" }; - private List dockerLogs = new List { "log1", "log2", "log3" }; - - List containers = new List(); - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertFailedTask() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); - - //Act - try - { - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - } - catch (InvalidOperationException) - { - - //Assert - Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); - - //Act and Assert - await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); - - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); - - //Act - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - - //Assert - Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); - - } - - private void Setup([CallerMemberName] string testName = "") - { - containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); - _hc = new TestHostContext(this, testName); - _ec = new Mock(); - serverQueue = new Mock(); - pagingLogger = new Mock(); - - _dockerManager = new Mock(); - _containerHookManager = new Mock(); - containerOperationProvider = new ContainerOperationProvider(); - - _hc.SetSingleton(_dockerManager.Object); - _hc.SetSingleton(serverQueue.Object); - _hc.SetSingleton(pagingLogger.Object); - - _hc.SetSingleton(_dockerManager.Object); - _hc.SetSingleton(_containerHookManager.Object); - - _ec.Setup(x => x.Global).Returns(new GlobalContext()); - - containerOperationProvider.Initialize(_hc); - } - } -} \ No newline at end of file