diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index 5fdcb2377..b6251112c 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -38,7 +38,6 @@ namespace GitHub.Runner.Worker var extensionManager = hostContext.GetService(); _commandExtensions = extensionManager.GetExtensions() ?? new List(); - } public void InitializeFiles(IExecutionContext context, ContainerInfo container) @@ -131,10 +130,128 @@ namespace GitHub.Runner.Worker public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - if (File.Exists(filePath)) + try { - // TODO Process this file + var text = File.ReadAllText(filePath) ?? string.Empty; + var index = 0; + var line = ReadLine(text, ref index); + while (line != null) + { + if (!string.IsNullOrEmpty(line)) + { + var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); + var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); + + // Normal style NAME=VALUE + if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + { + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(line)) + { + throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty"); + } + SetEnvironmentVariable(context, split[0], split[1]); + } + // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + { + var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + { + throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty and delimiter must not be empty"); + } + var name = split[0]; + var delimiter = split[1]; + var startIndex = index; // Start index of the value (inclusive) + var endIndex = index; // End index of the value (exclusive) + var tempLine = ReadLine(text, ref index, out var newline); + while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) + { + if (tempLine == null) + { + throw new Exception($"Invalid environment variable value. Matching delimiter not found '{delimiter}'"); + } + endIndex = index - newline.Length; + tempLine = ReadLine(text, ref index, out newline); + } + + var value = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; + SetEnvironmentVariable(context, name, value); + } + else + { + throw new Exception($"Invalid environment variable format '{line}'"); + } + } + + line = ReadLine(text, ref index); + } } + catch (DirectoryNotFoundException) + { + context.Debug($"Environment variables file does not exist '{filePath}'"); + } + catch (FileNotFoundException) + { + context.Debug($"Environment variables file does not exist '{filePath}'"); + } + catch (Exception ex) + { + context.Error($"Failed to read environment variables file '{filePath}'"); + context.Error(ex); + } + } + + private static void SetEnvironmentVariable( + IExecutionContext context, + string name, + string value) + { + context.Global.EnvironmentVariables[name] = value; + context.SetEnvContext(name, value); + context.Debug($"{name}='{value}'"); + } + + private static string ReadLine( + string text, + ref int index) + { + return ReadLine(text, ref index, out _); + } + + private static string ReadLine( + string text, + ref int index, + out string newline) + { + if (index >= text.Length) + { + newline = null; + return null; + } + + var originalIndex = index; + var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); + if (lfIndex < 0) + { + index = text.Length; + newline = null; + return text.Substring(originalIndex); + } + +#if OS_WINDOWS + var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); + if (crLFIndex >= 0 && crLFIndex < lfIndex) + { + index = crLFIndex + 2; // Skip over CRLF + newline = "\r\n"; + return text.Substring(originalIndex, crLFIndex - originalIndex); + } +#endif + + index = lfIndex + 1; // Skip over LF + newline = "\n"; + return text.Substring(originalIndex, lfIndex - originalIndex); } } } diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index 0e4b6d472..4eb98f770 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -41,5 +41,17 @@ namespace GitHub.Runner.Worker } } } + + public GitHubContext ShallowCopy() + { + var copy = new GitHubContext(); + + foreach (var pair in this) + { + copy[pair.Key] = pair.Value; + } + + return copy; + } } } \ No newline at end of file diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index 254277112..d7f92b0c0 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -32,9 +32,6 @@ namespace GitHub.Runner.Worker.Handlers ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.NotNull(Data.Steps, nameof(Data.Steps)); - var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; - ArgUtil.NotNull(githubContext, nameof(githubContext)); - // Resolve action steps var actionSteps = Data.Steps; @@ -56,14 +53,6 @@ namespace GitHub.Runner.Worker.Handlers childScopeName = $"__{Guid.NewGuid()}"; } - // Copy the github context so that we don't modify the original pointer - // We can't use PipelineContextData.Clone() since that creates a null pointer exception for copying a GitHubContext - var compositeGitHubContext = new GitHubContext(); - foreach (var pair in githubContext) - { - compositeGitHubContext[pair.Key] = pair.Value; - } - foreach (Pipelines.ActionStep actionStep in actionSteps) { var actionRunner = HostContext.CreateService(); @@ -73,8 +62,13 @@ namespace GitHub.Runner.Worker.Handlers var step = ExecutionContext.CreateCompositeStep(childScopeName, actionRunner, inputsData, Environment); + // Shallow copy github context + var gitHubContext = step.ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(gitHubContext, nameof(gitHubContext)); + gitHubContext = gitHubContext.ShallowCopy(); + step.ExecutionContext.ExpressionValues["github"] = gitHubContext; + // Set GITHUB_ACTION_PATH - step.ExecutionContext.ExpressionValues["github"] = compositeGitHubContext; step.ExecutionContext.SetGitHubContext("action_path", ActionDirectory); compositeSteps.Add(step); diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 58845206b..c0ab4f339 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -103,6 +103,12 @@ namespace GitHub.Runner.Worker bool evaluateStepEnvFailed = false; if (step is IActionRunner actionStep) { + // Shallow copy github context + var gitHubContext = step.ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(gitHubContext, nameof(gitHubContext)); + gitHubContext = gitHubContext.ShallowCopy(); + step.ExecutionContext.ExpressionValues["github"] = gitHubContext; + // Set GITHUB_ACTION step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); diff --git a/src/Test/L0/Worker/SetEnvFileCommandL0.cs b/src/Test/L0/Worker/SetEnvFileCommandL0.cs new file mode 100644 index 000000000..c655fb676 --- /dev/null +++ b/src/Test/L0/Worker/SetEnvFileCommandL0.cs @@ -0,0 +1,390 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; +using Moq; +using Xunit; +using DTWebApi = GitHub.DistributedTask.WebApi; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class SetEnvFileCommandL0 + { + private Mock _executionContext; + private List> _issues; + private string _rootDirectory; + private SetEnvFileCommand _setEnvFileCommand; + private ITraceWriter _trace; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_DirectoryNotFound() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "directory-not-found", "env"); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _executionContext.Object.Global.EnvironmentVariables.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_NotFound() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "file-not-found"); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _executionContext.Object.Global.EnvironmentVariables.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_EmptyFile() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "empty-file"); + var content = new List(); + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _executionContext.Object.Global.EnvironmentVariables.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_ENV=MY VALUE", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("MY VALUE", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_SkipEmptyLines() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + string.Empty, + "MY_ENV=my value", + string.Empty, + "MY_ENV_2=my second value", + string.Empty, + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(2, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("my value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + Assert.Equal("my second value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_2"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_EmptyValue() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple-empty-value"); + var content = new List + { + "MY_ENV=", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal(string.Empty, _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_MultipleValues() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_ENV=my value", + "MY_ENV_2=", + "MY_ENV_3=my third value", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("my value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + Assert.Equal(string.Empty, _executionContext.Object.Global.EnvironmentVariables["MY_ENV_2"]); + Assert.Equal("my third value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_SpecialCharacters() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_ENV==abc", + "MY_ENV_2=def=ghi", + "MY_ENV_3=jkl=", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("=abc", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + Assert.Equal("def=ghi", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_2"]); + Assert.Equal("jkl=", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Heredoc() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_ENV< + { + "MY_ENV< + { + string.Empty, + "MY_ENV< + { + "MY_ENV<<=EOF", + "hello", + "one", + "=EOF", + "MY_ENV_2<< + { + "MY_ENV< content, + string newline = null) + { + if (string.IsNullOrEmpty(newline)) + { + newline = Environment.NewLine; + } + + var encoding = new UTF8Encoding(true); // Emit BOM + var contentStr = string.Join(newline, content); + File.WriteAllText(path, contentStr, encoding); + } + + private TestHostContext Setup([CallerMemberName] string name = "") + { + _issues = new List>(); + + var hostContext = new TestHostContext(this, name); + + // Trace + _trace = hostContext.GetTrace(); + + // Directory for test data + var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); + ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); + Directory.CreateDirectory(workDirectory); + _rootDirectory = Path.Combine(workDirectory, nameof(SetEnvFileCommandL0)); + Directory.CreateDirectory(_rootDirectory); + + // Execution context + _executionContext = new Mock(); + _executionContext.Setup(x => x.Global) + .Returns(new GlobalContext + { + EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), + WriteDebug = true, + }); + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, string logMessage) => + { + _issues.Add(new Tuple(issue, logMessage)); + var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; + _trace.Info($"Issue '{issue.Type}': {message}"); + }); + _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + _trace.Info($"{tag}{message}"); + }); + + // SetEnvFileCommand + _setEnvFileCommand = new SetEnvFileCommand(); + _setEnvFileCommand.Initialize(hostContext); + + return hostContext; + } + } +} diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 3d97b0110..86843a083 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -44,7 +44,7 @@ namespace GitHub.Runner.Common.Tests.Worker _contexts = new DictionaryContextData(); _jobContext = new JobContext(); - _contexts["github"] = new DictionaryContextData(); + _contexts["github"] = new GitHubContext(); _contexts["runner"] = new DictionaryContextData(); _contexts["job"] = _jobContext; _ec.Setup(x => x.ExpressionValues).Returns(_contexts); @@ -602,7 +602,12 @@ namespace GitHub.Runner.Common.Tests.Worker var stepContext = new Mock(); stepContext.SetupAllProperties(); stepContext.Setup(x => x.Global).Returns(() => _ec.Object.Global); - stepContext.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); + var expressionValues = new DictionaryContextData(); + foreach (var pair in _ec.Object.ExpressionValues) + { + expressionValues[pair.Key] = pair.Value; + } + stepContext.Setup(x => x.ExpressionValues).Returns(expressionValues); stepContext.Setup(x => x.ExpressionFunctions).Returns(new List()); stepContext.Setup(x => x.JobContext).Returns(_jobContext); stepContext.Setup(x => x.ContextName).Returns(step.Object.Action.ContextName);