using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker.Container; using GitHub.Runner.Common; using GitHub.Runner.Sdk; using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Text; namespace GitHub.Runner.Worker { [ServiceLocator(Default = typeof(FileCommandManager))] public interface IFileCommandManager : IRunnerService { void InitializeFiles(IExecutionContext context, ContainerInfo container); void ProcessFiles(IExecutionContext context, ContainerInfo container); } public sealed class FileCommandManager : RunnerService, IFileCommandManager { private const string _folderName = "_runner_file_commands"; private List _commandExtensions; private string _fileSuffix = String.Empty; private string _fileCommandDirectory; private Tracing _trace; public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _trace = HostContext.GetTrace(nameof(FileCommandManager)); _fileCommandDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), _folderName); if (!Directory.Exists(_fileCommandDirectory)) { Directory.CreateDirectory(_fileCommandDirectory); } var extensionManager = hostContext.GetService(); _commandExtensions = extensionManager.GetExtensions() ?? new List(); } public void InitializeFiles(IExecutionContext context, ContainerInfo container) { var oldSuffix = _fileSuffix; _fileSuffix = Guid.NewGuid().ToString(); foreach (var fileCommand in _commandExtensions) { var oldPath = Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + oldSuffix); if (oldSuffix != String.Empty && File.Exists(oldPath)) { TryDeleteFile(oldPath); } var newPath = Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix); TryDeleteFile(newPath); File.Create(newPath).Dispose(); var pathToSet = container != null ? container.TranslateToContainerPath(newPath) : newPath; context.SetGitHubContext(fileCommand.ContextName, pathToSet); } } public void ProcessFiles(IExecutionContext context, ContainerInfo container) { foreach (var fileCommand in _commandExtensions) { try { fileCommand.ProcessCommand(context, Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix),container); } catch (Exception ex) { context.Error($"Unable to process file command '{fileCommand.ContextName}' successfully."); context.Error(ex); context.CommandResult = TaskResult.Failed; } } } private bool TryDeleteFile(string path) { if (!File.Exists(path)) { return true; } try { File.Delete(path); } catch (Exception e) { _trace.Warning($"Unable to delete file {path} for reason: {e.ToString()}"); return false; } return true; } } public interface IFileCommandExtension : IExtension { string ContextName { get; } string FilePrefix { get; } void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container); } public sealed class AddPathFileCommand : RunnerService, IFileCommandExtension { public string ContextName => "path"; public string FilePrefix => "add_path_"; public Type ExtensionType => typeof(IFileCommandExtension); public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { if (File.Exists(filePath)) { var lines = File.ReadAllLines(filePath, Encoding.UTF8); foreach(var line in lines) { if (line == string.Empty) { continue; } context.Global.PrependPath.RemoveAll(x => string.Equals(x, line, StringComparison.CurrentCulture)); context.Global.PrependPath.Add(line); } } } } public sealed class SetEnvFileCommand : RunnerService, IFileCommandExtension { public string ContextName => "env"; public string FilePrefix => "set_env_"; public Type ExtensionType => typeof(IFileCommandExtension); public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { try { var text = File.ReadAllText(filePath) ?? string.Empty; var index = 0; var line = ReadLine(text, ref index); while (line != null) { if (!string.IsNullOrEmpty(line)) { var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); // Normal style NAME=VALUE if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) { var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); if (string.IsNullOrEmpty(line)) { throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty"); } SetEnvironmentVariable(context, split[0], split[1]); } // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) { var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) { throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty and delimiter must not be empty"); } var name = split[0]; var delimiter = split[1]; var startIndex = index; // Start index of the value (inclusive) var endIndex = index; // End index of the value (exclusive) var tempLine = ReadLine(text, ref index, out var newline); while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) { if (tempLine == null) { throw new Exception($"Invalid environment variable value. Matching delimiter not found '{delimiter}'"); } if (newline == null) { throw new Exception($"Invalid environment variable value. EOF marker missing new line."); } endIndex = index - newline.Length; tempLine = ReadLine(text, ref index, out newline); } var value = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; SetEnvironmentVariable(context, name, value); } else { throw new Exception($"Invalid environment variable format '{line}'"); } } line = ReadLine(text, ref index); } } catch (DirectoryNotFoundException) { context.Debug($"Environment variables file does not exist '{filePath}'"); } catch (FileNotFoundException) { context.Debug($"Environment variables file does not exist '{filePath}'"); } } private static void SetEnvironmentVariable( IExecutionContext context, string name, string value) { context.Global.EnvironmentVariables[name] = value; context.SetEnvContext(name, value); context.Debug($"{name}='{value}'"); } private static string ReadLine( string text, ref int index) { return ReadLine(text, ref index, out _); } private static string ReadLine( string text, ref int index, out string newline) { if (index >= text.Length) { newline = null; return null; } var originalIndex = index; var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); if (lfIndex < 0) { index = text.Length; newline = null; return text.Substring(originalIndex); } #if OS_WINDOWS var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); if (crLFIndex >= 0 && crLFIndex < lfIndex) { index = crLFIndex + 2; // Skip over CRLF newline = "\r\n"; return text.Substring(originalIndex, crLFIndex - originalIndex); } #endif index = lfIndex + 1; // Skip over LF newline = "\n"; return text.Substring(originalIndex, lfIndex - originalIndex); } } public sealed class CreateStepSummaryCommand : RunnerService, IFileCommandExtension { public const int AttachmentSizeLimit = 1024 * 1024; public string ContextName => "step_summary"; public string FilePrefix => "step_summary_"; public Type ExtensionType => typeof(IFileCommandExtension); public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { if (String.IsNullOrEmpty(filePath) || !File.Exists(filePath)) { Trace.Info($"Step Summary file ({filePath}) does not exist; skipping attachment upload"); return; } try { var fileSize = new FileInfo(filePath).Length; if (fileSize == 0) { Trace.Info($"Step Summary file ({filePath}) is empty; skipping attachment upload"); return; } if (fileSize > AttachmentSizeLimit) { context.Error(String.Format(Constants.Runner.UnsupportedSummarySize, AttachmentSizeLimit / 1024, fileSize / 1024)); Trace.Info($"Step Summary file ({filePath}) is too large ({fileSize} bytes); skipping attachment upload"); return; } Trace.Verbose($"Step Summary file exists: {filePath} and has a file size of {fileSize} bytes"); var scrubbedFilePath = filePath + "-scrubbed"; using (var streamReader = new StreamReader(filePath)) using (var streamWriter = new StreamWriter(scrubbedFilePath)) { string line; while ((line = streamReader.ReadLine()) != null) { var maskedLine = HostContext.SecretMasker.MaskSecrets(line); streamWriter.WriteLine(maskedLine); } } var attachmentName = context.Id.ToString(); Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); // Attachments must be added to the parent context (job), not the current context (step) context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); } catch (Exception e) { Trace.Error($"Error while processing file ({filePath}): {e}"); context.Error($"Failed to create step summary using 'GITHUB_STEP_SUMMARY': {e.Message}"); } } } }