diff --git a/src/Runner.Common/ExtensionManager.cs b/src/Runner.Common/ExtensionManager.cs index c53fc3647..9b7171c73 100644 --- a/src/Runner.Common/ExtensionManager.cs +++ b/src/Runner.Common/ExtensionManager.cs @@ -56,6 +56,10 @@ namespace GitHub.Runner.Common Add(extensions, "GitHub.Runner.Worker.EndGroupCommandExtension, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.EchoCommandExtension, Runner.Worker"); break; + case "GitHub.Runner.Worker.IFileCommandExtension": + Add(extensions, "GitHub.Runner.Worker.AddPathFileCommand, Runner.Worker"); + Add(extensions, "GitHub.Runner.Worker.SetEnvFileCommand, Runner.Worker"); + break; default: // This should never happen. throw new NotSupportedException($"Unexpected extension type: '{typeof(T).FullName}'"); diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index e3e0f551b..7d069bf97 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -16,6 +16,7 @@ namespace GitHub.Runner.Common // logging and console Task AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, CancellationToken cancellationToken); + Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long startLine, CancellationToken cancellationToken); Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); @@ -79,6 +80,12 @@ namespace GitHub.Runner.Common return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, cancellationToken: cancellationToken); } + public Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long startLine, CancellationToken cancellationToken) + { + CheckConnection(); + return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, startLine, cancellationToken: cancellationToken); + } + public Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken) { CheckConnection(); diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 787daeffd..5cabca2a9 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -18,7 +18,7 @@ namespace GitHub.Runner.Common event EventHandler JobServerQueueThrottling; Task ShutdownAsync(); void Start(Pipelines.AgentJobRequestMessage jobRequest); - void QueueWebConsoleLine(Guid stepRecordId, string line); + void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); } @@ -155,10 +155,10 @@ namespace GitHub.Runner.Common Trace.Info("All queue process tasks have been stopped, and all queues are drained."); } - public void QueueWebConsoleLine(Guid stepRecordId, string line) + public void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber) { Trace.Verbose("Enqueue web console line queue: {0}", line); - _webConsoleLineQueue.Enqueue(new ConsoleLineInfo(stepRecordId, line)); + _webConsoleLineQueue.Enqueue(new ConsoleLineInfo(stepRecordId, line, lineNumber)); } public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource) @@ -214,7 +214,7 @@ namespace GitHub.Runner.Common } // Group consolelines by timeline record of each step - Dictionary> stepsConsoleLines = new Dictionary>(); + Dictionary> stepsConsoleLines = new Dictionary>(); List stepRecordIds = new List(); // We need to keep lines in order int linesCounter = 0; ConsoleLineInfo lineInfo; @@ -222,7 +222,7 @@ namespace GitHub.Runner.Common { if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId)) { - stepsConsoleLines[lineInfo.StepRecordId] = new List(); + stepsConsoleLines[lineInfo.StepRecordId] = new List(); stepRecordIds.Add(lineInfo.StepRecordId); } @@ -232,7 +232,7 @@ namespace GitHub.Runner.Common lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}..."; } - stepsConsoleLines[lineInfo.StepRecordId].Add(lineInfo.Line); + stepsConsoleLines[lineInfo.StepRecordId].Add(new TimelineRecordLogLine(lineInfo.Line, lineInfo.LineNumber)); linesCounter++; // process at most about 500 lines of web console line during regular timer dequeue task. @@ -247,13 +247,13 @@ namespace GitHub.Runner.Common { // Split consolelines into batch, each batch will container at most 100 lines. int batchCounter = 0; - List> batchedLines = new List>(); + List> batchedLines = new List>(); foreach (var line in stepsConsoleLines[stepRecordId]) { var currentBatch = batchedLines.ElementAtOrDefault(batchCounter); if (currentBatch == null) { - batchedLines.Add(new List()); + batchedLines.Add(new List()); currentBatch = batchedLines.ElementAt(batchCounter); } @@ -275,7 +275,6 @@ namespace GitHub.Runner.Common { Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run"); batchedLines = batchedLines.TakeLast(2).ToList(); - batchedLines[0].Insert(0, "..."); } int errorCount = 0; @@ -284,7 +283,15 @@ namespace GitHub.Runner.Common try { // we will not requeue failed batch, since the web console lines are time sensitive. - await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch, default(CancellationToken)); + if (batch[0].LineNumber.HasValue) + { + await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber.Value, default(CancellationToken)); + } + else + { + await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), default(CancellationToken)); + } + if (_firstConsoleOutputs) { HostContext.WritePerfCounter($"WorkerJobServerQueueAppendFirstConsoleOutput_{_planId.ToString()}"); @@ -653,13 +660,15 @@ namespace GitHub.Runner.Common internal class ConsoleLineInfo { - public ConsoleLineInfo(Guid recordId, string line) + public ConsoleLineInfo(Guid recordId, string line, long? lineNumber) { this.StepRecordId = recordId; this.Line = line; + this.LineNumber = lineNumber; } public Guid StepRecordId { get; set; } public string Line { get; set; } + public long? LineNumber { get; set; } } } diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 8a12b9561..d9387286a 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -145,6 +145,10 @@ namespace GitHub.Runner.Worker stepHost = containerStepHost; } + // Setup File Command Manager + var fileCommandManager = HostContext.CreateService(); + fileCommandManager.InitializeFiles(ExecutionContext, null); + // Load the inputs. ExecutionContext.Debug("Loading inputs"); var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); @@ -238,7 +242,15 @@ namespace GitHub.Runner.Worker handler.PrintActionDetails(Stage); // Run the task. - await handler.RunAsync(Stage); + try + { + await handler.RunAsync(Stage); + } + finally + { + fileCommandManager.ProcessFiles(ExecutionContext, ExecutionContext.Global.Container); + } + } public bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 224b33b0e..46f516999 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -717,7 +717,8 @@ namespace GitHub.Runner.Worker } } - _jobServerQueue.QueueWebConsoleLine(_record.Id, msg); + _jobServerQueue.QueueWebConsoleLine(_record.Id, msg, totalLines); + return totalLines; } diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs new file mode 100644 index 000000000..aea76f10a --- /dev/null +++ b/src/Runner.Worker/FileCommandManager.cs @@ -0,0 +1,262 @@ +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace GitHub.Runner.Worker +{ + [ServiceLocator(Default = typeof(FileCommandManager))] + public interface IFileCommandManager : IRunnerService + { + void InitializeFiles(IExecutionContext context, ContainerInfo container); + void ProcessFiles(IExecutionContext context, ContainerInfo container); + + } + + public sealed class FileCommandManager : RunnerService, IFileCommandManager + { + private const string _folderName = "_runner_file_commands"; + private List _commandExtensions; + private string _fileSuffix = String.Empty; + private string _fileCommandDirectory; + private Tracing _trace; + + public override void Initialize(IHostContext hostContext) + { + base.Initialize(hostContext); + _trace = HostContext.GetTrace(nameof(FileCommandManager)); + + _fileCommandDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), _folderName); + if (!Directory.Exists(_fileCommandDirectory)) + { + Directory.CreateDirectory(_fileCommandDirectory); + } + + var extensionManager = hostContext.GetService(); + _commandExtensions = extensionManager.GetExtensions() ?? new List(); + } + + public void InitializeFiles(IExecutionContext context, ContainerInfo container) + { + var oldSuffix = _fileSuffix; + _fileSuffix = Guid.NewGuid().ToString(); + foreach (var fileCommand in _commandExtensions) + { + var oldPath = Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + oldSuffix); + if (oldSuffix != String.Empty && File.Exists(oldPath)) + { + TryDeleteFile(oldPath); + } + + var newPath = Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix); + TryDeleteFile(newPath); + File.Create(newPath).Dispose(); + + var pathToSet = container != null ? container.TranslateToContainerPath(newPath) : newPath; + context.SetGitHubContext(fileCommand.ContextName, pathToSet); + } + } + + public void ProcessFiles(IExecutionContext context, ContainerInfo container) + { + foreach (var fileCommand in _commandExtensions) + { + try + { + fileCommand.ProcessCommand(context, Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix),container); + } + catch (Exception ex) + { + context.Error($"Unable to process file command '{fileCommand.ContextName}' successfully."); + context.Error(ex); + context.CommandResult = TaskResult.Failed; + } + } + } + + private bool TryDeleteFile(string path) + { + if (!File.Exists(path)) + { + return true; + } + try + { + File.Delete(path); + } + catch (Exception e) + { + _trace.Warning($"Unable to delete file {path} for reason: {e.ToString()}"); + return false; + } + return true; + } + } + + public interface IFileCommandExtension : IExtension + { + string ContextName { get; } + string FilePrefix { get; } + + void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container); + } + + public sealed class AddPathFileCommand : RunnerService, IFileCommandExtension + { + public string ContextName => "path"; + public string FilePrefix => "add_path_"; + + public Type ExtensionType => typeof(IFileCommandExtension); + + public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) + { + if (File.Exists(filePath)) + { + var lines = File.ReadAllLines(filePath, Encoding.UTF8); + foreach(var line in lines) + { + if (line == string.Empty) + { + continue; + } + context.Global.PrependPath.RemoveAll(x => string.Equals(x, line, StringComparison.CurrentCulture)); + context.Global.PrependPath.Add(line); + } + } + } + } + + public sealed class SetEnvFileCommand : RunnerService, IFileCommandExtension + { + public string ContextName => "env"; + public string FilePrefix => "set_env_"; + + public Type ExtensionType => typeof(IFileCommandExtension); + + public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) + { + try + { + var text = File.ReadAllText(filePath) ?? string.Empty; + var index = 0; + var line = ReadLine(text, ref index); + while (line != null) + { + if (!string.IsNullOrEmpty(line)) + { + var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); + var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); + + // Normal style NAME=VALUE + if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + { + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(line)) + { + throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty"); + } + SetEnvironmentVariable(context, split[0], split[1]); + } + // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + { + var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + { + throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty and delimiter must not be empty"); + } + var name = split[0]; + var delimiter = split[1]; + var startIndex = index; // Start index of the value (inclusive) + var endIndex = index; // End index of the value (exclusive) + var tempLine = ReadLine(text, ref index, out var newline); + while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) + { + if (tempLine == null) + { + throw new Exception($"Invalid environment variable value. Matching delimiter not found '{delimiter}'"); + } + endIndex = index - newline.Length; + tempLine = ReadLine(text, ref index, out newline); + } + + var value = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; + SetEnvironmentVariable(context, name, value); + } + else + { + throw new Exception($"Invalid environment variable format '{line}'"); + } + } + + line = ReadLine(text, ref index); + } + } + catch (DirectoryNotFoundException) + { + context.Debug($"Environment variables file does not exist '{filePath}'"); + } + catch (FileNotFoundException) + { + context.Debug($"Environment variables file does not exist '{filePath}'"); + } + } + + private static void SetEnvironmentVariable( + IExecutionContext context, + string name, + string value) + { + context.Global.EnvironmentVariables[name] = value; + context.SetEnvContext(name, value); + context.Debug($"{name}='{value}'"); + } + + private static string ReadLine( + string text, + ref int index) + { + return ReadLine(text, ref index, out _); + } + + private static string ReadLine( + string text, + ref int index, + out string newline) + { + if (index >= text.Length) + { + newline = null; + return null; + } + + var originalIndex = index; + var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); + if (lfIndex < 0) + { + index = text.Length; + newline = null; + return text.Substring(originalIndex); + } + +#if OS_WINDOWS + var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); + if (crLFIndex >= 0 && crLFIndex < lfIndex) + { + index = crLFIndex + 2; // Skip over CRLF + newline = "\r\n"; + return text.Substring(originalIndex, crLFIndex - originalIndex); + } +#endif + + index = lfIndex + 1; // Skip over LF + newline = "\n"; + return text.Substring(originalIndex, lfIndex - originalIndex); + } + } +} diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index 1ecca19f0..5079206cf 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -6,18 +6,20 @@ namespace GitHub.Runner.Worker { public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextData { - private readonly HashSet _contextEnvWhitelist = new HashSet(StringComparer.OrdinalIgnoreCase) + private readonly HashSet _contextEnvAllowlist = new HashSet(StringComparer.OrdinalIgnoreCase) { "action", "action_path", "actor", "api_url", "base_ref", + "env", "event_name", "event_path", "graphql_url", "head_ref", "job", + "path", "ref", "repository", "repository_owner", @@ -33,11 +35,23 @@ namespace GitHub.Runner.Worker { foreach (var data in this) { - if (_contextEnvWhitelist.Contains(data.Key) && data.Value is StringContextData value) + if (_contextEnvAllowlist.Contains(data.Key) && data.Value is StringContextData value) { yield return new KeyValuePair($"GITHUB_{data.Key.ToUpperInvariant()}", value); } } } + + public GitHubContext ShallowCopy() + { + var copy = new GitHubContext(); + + foreach (var pair in this) + { + copy[pair.Key] = pair.Value; + } + + return copy; + } } } \ No newline at end of file diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index 254277112..d7f92b0c0 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -32,9 +32,6 @@ namespace GitHub.Runner.Worker.Handlers ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.NotNull(Data.Steps, nameof(Data.Steps)); - var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; - ArgUtil.NotNull(githubContext, nameof(githubContext)); - // Resolve action steps var actionSteps = Data.Steps; @@ -56,14 +53,6 @@ namespace GitHub.Runner.Worker.Handlers childScopeName = $"__{Guid.NewGuid()}"; } - // Copy the github context so that we don't modify the original pointer - // We can't use PipelineContextData.Clone() since that creates a null pointer exception for copying a GitHubContext - var compositeGitHubContext = new GitHubContext(); - foreach (var pair in githubContext) - { - compositeGitHubContext[pair.Key] = pair.Value; - } - foreach (Pipelines.ActionStep actionStep in actionSteps) { var actionRunner = HostContext.CreateService(); @@ -73,8 +62,13 @@ namespace GitHub.Runner.Worker.Handlers var step = ExecutionContext.CreateCompositeStep(childScopeName, actionRunner, inputsData, Environment); + // Shallow copy github context + var gitHubContext = step.ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(gitHubContext, nameof(gitHubContext)); + gitHubContext = gitHubContext.ShallowCopy(); + step.ExecutionContext.ExpressionValues["github"] = gitHubContext; + // Set GITHUB_ACTION_PATH - step.ExecutionContext.ExpressionValues["github"] = compositeGitHubContext; step.ExecutionContext.SetGitHubContext("action_path", ActionDirectory); compositeSteps.Add(step); diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index 6e8624510..b25383459 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -161,16 +161,21 @@ namespace GitHub.Runner.Worker.Handlers Directory.CreateDirectory(tempHomeDirectory); this.Environment["HOME"] = tempHomeDirectory; + var tempFileCommandDirectory = Path.Combine(tempDirectory, "_runner_file_commands"); + ArgUtil.Directory(tempFileCommandDirectory, nameof(tempFileCommandDirectory)); + var tempWorkflowDirectory = Path.Combine(tempDirectory, "_github_workflow"); ArgUtil.Directory(tempWorkflowDirectory, nameof(tempWorkflowDirectory)); container.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock")); container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home")); container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow")); + container.MountVolumes.Add(new MountVolume(tempFileCommandDirectory, "/github/file_commands")); container.MountVolumes.Add(new MountVolume(defaultWorkingDirectory, "/github/workspace")); container.AddPathTranslateMapping(tempHomeDirectory, "/github/home"); container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow"); + container.AddPathTranslateMapping(tempFileCommandDirectory, "/github/file_commands"); container.AddPathTranslateMapping(defaultWorkingDirectory, "/github/workspace"); container.ContainerWorkDirectory = "/github/workspace"; diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 58845206b..c0ab4f339 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -103,6 +103,12 @@ namespace GitHub.Runner.Worker bool evaluateStepEnvFailed = false; if (step is IActionRunner actionStep) { + // Shallow copy github context + var gitHubContext = step.ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(gitHubContext, nameof(gitHubContext)); + gitHubContext = gitHubContext.ShallowCopy(); + step.ExecutionContext.ExpressionValues["github"] = gitHubContext; + // Set GITHUB_ACTION step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); diff --git a/src/Sdk/DTWebApi/WebApi/TaskHttpClient.cs b/src/Sdk/DTWebApi/WebApi/TaskHttpClient.cs index fcc1ab6ce..6a5515552 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskHttpClient.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskHttpClient.cs @@ -50,7 +50,7 @@ namespace GitHub.DistributedTask.WebApi : base(baseUrl, pipeline, disposeHandler) { } - + public Task AppendTimelineRecordFeedAsync( Guid scopeIdentifier, String planType, @@ -91,6 +91,28 @@ namespace GitHub.DistributedTask.WebApi userState, cancellationToken); } + + public Task AppendTimelineRecordFeedAsync( + Guid scopeIdentifier, + String planType, + Guid planId, + Guid timelineId, + Guid recordId, + Guid stepId, + IList lines, + long startLine, + CancellationToken cancellationToken = default(CancellationToken), + Object userState = null) + { + return AppendTimelineRecordFeedAsync(scopeIdentifier, + planType, + planId, + timelineId, + recordId, + new TimelineRecordFeedLinesWrapper(stepId, lines, startLine), + userState, + cancellationToken); + } public async Task RaisePlanEventAsync( Guid scopeIdentifier, diff --git a/src/Sdk/DTWebApi/WebApi/TimelineRecordFeedLinesWrapper.cs b/src/Sdk/DTWebApi/WebApi/TimelineRecordFeedLinesWrapper.cs index c628852bd..edaaf314a 100644 --- a/src/Sdk/DTWebApi/WebApi/TimelineRecordFeedLinesWrapper.cs +++ b/src/Sdk/DTWebApi/WebApi/TimelineRecordFeedLinesWrapper.cs @@ -20,6 +20,12 @@ namespace GitHub.DistributedTask.WebApi this.Count = lines.Count; } + public TimelineRecordFeedLinesWrapper(Guid stepId, IList lines, Int64 startLine) + : this(stepId, lines) + { + this.StartLine = startLine; + } + [DataMember(Order = 0)] public Int32 Count { get; private set; } @@ -31,5 +37,8 @@ namespace GitHub.DistributedTask.WebApi [DataMember(EmitDefaultValue = false)] public Guid StepId { get; set; } + + [DataMember (EmitDefaultValue = false)] + public Int64? StartLine { get; private set; } } } diff --git a/src/Sdk/DTWebApi/WebApi/TimelineRecordLogLine.cs b/src/Sdk/DTWebApi/WebApi/TimelineRecordLogLine.cs new file mode 100644 index 000000000..2761390c9 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/TimelineRecordLogLine.cs @@ -0,0 +1,29 @@ +using System; +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public sealed class TimelineRecordLogLine + { + public TimelineRecordLogLine(String line, long? lineNumber) + { + this.Line = line; + this.LineNumber = lineNumber; + } + + [DataMember] + public String Line + { + get; + set; + } + + [DataMember (EmitDefaultValue = false)] + public long? LineNumber + { + get; + set; + } + } +} diff --git a/src/Test/L0/ServiceInterfacesL0.cs b/src/Test/L0/ServiceInterfacesL0.cs index 63c01b17d..b3535e40a 100644 --- a/src/Test/L0/ServiceInterfacesL0.cs +++ b/src/Test/L0/ServiceInterfacesL0.cs @@ -60,6 +60,7 @@ namespace GitHub.Runner.Common.Tests { typeof(IActionCommandExtension), typeof(IExecutionContext), + typeof(IFileCommandExtension), typeof(IHandler), typeof(IJobExtension), typeof(IStep), diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index c5c6426e4..371a6253b 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -32,6 +32,8 @@ namespace GitHub.Runner.Common.Tests.Worker private TestHostContext _hc; private ActionRunner _actionRunner; private IActionManifestManager _actionManifestManager; + private Mock _fileCommandManager; + private DictionaryContextData _context = new DictionaryContextData(); [Fact] @@ -362,6 +364,7 @@ namespace GitHub.Runner.Common.Tests.Worker _handlerFactory = new Mock(); _defaultStepHost = new Mock(); _actionManifestManager = new ActionManifestManager(); + _fileCommandManager = new Mock(); _actionManifestManager.Initialize(_hc); var githubContext = new GitHubContext(); @@ -394,6 +397,8 @@ namespace GitHub.Runner.Common.Tests.Worker _hc.EnqueueInstance(_defaultStepHost.Object); + _hc.EnqueueInstance(_fileCommandManager.Object); + // Instance to test. _actionRunner = new ActionRunner(); _actionRunner.Initialize(_hc); diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 0efd4bfe5..39dbb2dda 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -116,7 +116,7 @@ namespace GitHub.Runner.Common.Tests.Worker var pagingLogger = new Mock(); var jobServerQueue = new Mock(); jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); - jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny())).Callback((Guid id, string msg) => { hc.GetTrace().Info(msg); }); + jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny(),It.IsAny())).Callback((Guid id, string msg, long? lineNumber) => { hc.GetTrace().Info(msg); }); hc.EnqueueInstance(pagingLogger.Object); hc.SetSingleton(jobServerQueue.Object); @@ -137,7 +137,7 @@ namespace GitHub.Runner.Common.Tests.Worker ec.Complete(); - jobServerQueue.Verify(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny()), Times.Exactly(10)); + jobServerQueue.Verify(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(10)); } } @@ -171,7 +171,7 @@ namespace GitHub.Runner.Common.Tests.Worker var pagingLogger5 = new Mock(); var jobServerQueue = new Mock(); jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); - jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny())).Callback((Guid id, string msg) => { hc.GetTrace().Info(msg); }); + jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny(), It.IsAny())).Callback((Guid id, string msg, long? lineNumber) => { hc.GetTrace().Info(msg); }); var actionRunner1 = new ActionRunner(); actionRunner1.Initialize(hc); @@ -269,7 +269,7 @@ namespace GitHub.Runner.Common.Tests.Worker var pagingLogger5 = new Mock(); var jobServerQueue = new Mock(); jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); - jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny())).Callback((Guid id, string msg) => { hc.GetTrace().Info(msg); }); + jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny(), It.IsAny())).Callback((Guid id, string msg, long? lineNumber) => { hc.GetTrace().Info(msg); }); var actionRunner1 = new ActionRunner(); actionRunner1.Initialize(hc); diff --git a/src/Test/L0/Worker/SetEnvFileCommandL0.cs b/src/Test/L0/Worker/SetEnvFileCommandL0.cs new file mode 100644 index 000000000..c655fb676 --- /dev/null +++ b/src/Test/L0/Worker/SetEnvFileCommandL0.cs @@ -0,0 +1,390 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; +using Moq; +using Xunit; +using DTWebApi = GitHub.DistributedTask.WebApi; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class SetEnvFileCommandL0 + { + private Mock _executionContext; + private List> _issues; + private string _rootDirectory; + private SetEnvFileCommand _setEnvFileCommand; + private ITraceWriter _trace; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_DirectoryNotFound() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "directory-not-found", "env"); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _executionContext.Object.Global.EnvironmentVariables.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_NotFound() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "file-not-found"); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _executionContext.Object.Global.EnvironmentVariables.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_EmptyFile() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "empty-file"); + var content = new List(); + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _executionContext.Object.Global.EnvironmentVariables.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_ENV=MY VALUE", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("MY VALUE", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_SkipEmptyLines() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + string.Empty, + "MY_ENV=my value", + string.Empty, + "MY_ENV_2=my second value", + string.Empty, + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(2, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("my value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + Assert.Equal("my second value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_2"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_EmptyValue() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple-empty-value"); + var content = new List + { + "MY_ENV=", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal(string.Empty, _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_MultipleValues() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_ENV=my value", + "MY_ENV_2=", + "MY_ENV_3=my third value", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("my value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + Assert.Equal(string.Empty, _executionContext.Object.Global.EnvironmentVariables["MY_ENV_2"]); + Assert.Equal("my third value", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Simple_SpecialCharacters() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_ENV==abc", + "MY_ENV_2=def=ghi", + "MY_ENV_3=jkl=", + }; + WriteContent(envFile, content); + _setEnvFileCommand.ProcessCommand(_executionContext.Object, envFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _executionContext.Object.Global.EnvironmentVariables.Count); + Assert.Equal("=abc", _executionContext.Object.Global.EnvironmentVariables["MY_ENV"]); + Assert.Equal("def=ghi", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_2"]); + Assert.Equal("jkl=", _executionContext.Object.Global.EnvironmentVariables["MY_ENV_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetEnvFileCommand_Heredoc() + { + using (var hostContext = Setup()) + { + var envFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_ENV< + { + "MY_ENV< + { + string.Empty, + "MY_ENV< + { + "MY_ENV<<=EOF", + "hello", + "one", + "=EOF", + "MY_ENV_2<< + { + "MY_ENV< content, + string newline = null) + { + if (string.IsNullOrEmpty(newline)) + { + newline = Environment.NewLine; + } + + var encoding = new UTF8Encoding(true); // Emit BOM + var contentStr = string.Join(newline, content); + File.WriteAllText(path, contentStr, encoding); + } + + private TestHostContext Setup([CallerMemberName] string name = "") + { + _issues = new List>(); + + var hostContext = new TestHostContext(this, name); + + // Trace + _trace = hostContext.GetTrace(); + + // Directory for test data + var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); + ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); + Directory.CreateDirectory(workDirectory); + _rootDirectory = Path.Combine(workDirectory, nameof(SetEnvFileCommandL0)); + Directory.CreateDirectory(_rootDirectory); + + // Execution context + _executionContext = new Mock(); + _executionContext.Setup(x => x.Global) + .Returns(new GlobalContext + { + EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), + WriteDebug = true, + }); + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, string logMessage) => + { + _issues.Add(new Tuple(issue, logMessage)); + var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; + _trace.Info($"Issue '{issue.Type}': {message}"); + }); + _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + _trace.Info($"{tag}{message}"); + }); + + // SetEnvFileCommand + _setEnvFileCommand = new SetEnvFileCommand(); + _setEnvFileCommand.Initialize(hostContext); + + return hostContext; + } + } +} diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 3d97b0110..86843a083 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -44,7 +44,7 @@ namespace GitHub.Runner.Common.Tests.Worker _contexts = new DictionaryContextData(); _jobContext = new JobContext(); - _contexts["github"] = new DictionaryContextData(); + _contexts["github"] = new GitHubContext(); _contexts["runner"] = new DictionaryContextData(); _contexts["job"] = _jobContext; _ec.Setup(x => x.ExpressionValues).Returns(_contexts); @@ -602,7 +602,12 @@ namespace GitHub.Runner.Common.Tests.Worker var stepContext = new Mock(); stepContext.SetupAllProperties(); stepContext.Setup(x => x.Global).Returns(() => _ec.Object.Global); - stepContext.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); + var expressionValues = new DictionaryContextData(); + foreach (var pair in _ec.Object.ExpressionValues) + { + expressionValues[pair.Key] = pair.Value; + } + stepContext.Setup(x => x.ExpressionValues).Returns(expressionValues); stepContext.Setup(x => x.ExpressionFunctions).Returns(new List()); stepContext.Setup(x => x.JobContext).Returns(_jobContext); stepContext.Setup(x => x.ContextName).Returns(step.Object.Action.ContextName);