GitHub Actions Runner

src/Sdk/DTPipelines/Pipelines/Validation/GraphValidator.cs

@@ -0,0 +1,186 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    internal static class GraphValidator
+    {
+        internal delegate String ErrorFormatter(String code, params Object[] values);
+
+        internal static void Validate<T>(
+            PipelineBuildContext context,
+            ValidationResult result,
+            Func<Object, String> getBaseRefName,
+            String graphName,
+            IList<T> nodes,
+            ErrorFormatter formatError) where T : class, IGraphNode
+        {
+            var unnamedNodes = new List<T>();
+            var startingNodes = new List<T>();
+            var knownNames = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
+            Boolean hasDuplicateName = false;
+            foreach (var node in nodes)
+            {
+                if (!String.IsNullOrEmpty(node.Name))
+                {
+                    if (!NameValidation.IsValid(node.Name, context.BuildOptions.AllowHyphenNames))
+                    {
+                        result.Errors.Add(new PipelineValidationError(PipelineConstants.NameInvalid, formatError(PipelineConstants.NameInvalid, graphName, node.Name)));
+                    }
+                    else if (!knownNames.Add(node.Name))
+                    {
+                        hasDuplicateName = true;
+                        result.Errors.Add(new PipelineValidationError(PipelineConstants.NameNotUnique, formatError(PipelineConstants.NameNotUnique, graphName, node.Name)));
+                    }
+                }
+                else
+                {
+                    unnamedNodes.Add(node);
+                }
+
+                if (node.DependsOn.Count == 0)
+                {
+                    startingNodes.Add(node);
+                }
+            }
+
+            Int32 nodeCounter = 1;
+            foreach (var unnamedNode in unnamedNodes)
+            {
+                var candidateName = getBaseRefName(nodeCounter);
+                while (!knownNames.Add(candidateName))
+                {
+                    nodeCounter++;
+                    candidateName = getBaseRefName(nodeCounter);
+                }
+
+                nodeCounter++;
+                unnamedNode.Name = candidateName;
+            }
+
+            // Now that we have generated default names we can validate and provide error messages
+            foreach (var node in nodes)
+            {
+                node.Validate(context, result);
+            }
+
+            if (startingNodes.Count == 0)
+            {
+                result.Errors.Add(new PipelineValidationError(PipelineConstants.StartingPointNotFound, formatError(PipelineConstants.StartingPointNotFound, graphName)));
+                return;
+            }
+
+            // Skip validating the graph if duplicate phase names
+            if (hasDuplicateName)
+            {
+                return;
+            }
+
+            var nodesToVisit = new Queue<T>(startingNodes);
+            var nodeLookup = nodes.ToDictionary(x => x.Name, StringComparer.OrdinalIgnoreCase);
+            var unsatisfiedDependencies = nodes.ToDictionary(x => x.Name, x => new List<String>(x.DependsOn), StringComparer.OrdinalIgnoreCase);
+            var visitedNames = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
+            while (nodesToVisit.Count > 0)
+            {
+                var currentPhase = nodesToVisit.Dequeue();
+
+                visitedNames.Add(currentPhase.Name);
+
+                // Now figure out which nodes would start as a result of this 
+                foreach (var nodeState in unsatisfiedDependencies)
+                {
+                    for (Int32 i = nodeState.Value.Count - 1; i >= 0; i--)
+                    {
+                        if (nodeState.Value[i].Equals(currentPhase.Name, StringComparison.OrdinalIgnoreCase))
+                        {
+                            nodeState.Value.RemoveAt(i);
+                            if (nodeState.Value.Count == 0)
+                            {
+                                nodesToVisit.Enqueue(nodeLookup[nodeState.Key]);
+                            }
+                        }
+                    }
+                }
+            }
+
+            // There are nodes which are never going to execute, which is generally caused by a cycle in the graph.
+            var unreachableNodeCount = nodes.Count - visitedNames.Count;
+            if (unreachableNodeCount > 0)
+            {
+                foreach (var unreachableNode in unsatisfiedDependencies.Where(x => x.Value.Count > 0))
+                {
+                    foreach (var unsatisifedDependency in unreachableNode.Value)
+                    {
+                        if (!nodeLookup.ContainsKey(unsatisifedDependency))
+                        {
+                            result.Errors.Add(new PipelineValidationError(PipelineConstants.DependencyNotFound, formatError(PipelineConstants.DependencyNotFound, graphName, unreachableNode.Key, unsatisifedDependency)));
+                        }
+                        else
+                        {
+                            result.Errors.Add(new PipelineValidationError(PipelineConstants.GraphContainsCycle, formatError(PipelineConstants.GraphContainsCycle, graphName, unreachableNode.Key, unsatisifedDependency)));
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Traverses a validated graph running a callback on each node in the order it would execute at runtime.
+        /// </summary>
+        /// <typeparam name="T">The type of graph node</typeparam>
+        /// <param name="nodes">The full set of nodes in the graph</param>
+        /// <param name="handleNode">A callback which is invoked for each node as execution would begin</param>
+        internal static void Traverse<T>(
+            IList<T> nodes, 
+            Action<T, ISet<String>> handleNode) where T : class, IGraphNode
+        {
+            var nodeLookup = nodes.ToDictionary(x => x.Name, x => new GraphTraversalState<T>(x), StringComparer.OrdinalIgnoreCase);
+            var pendingNodes = nodes.ToDictionary(x => x.Name, x => new List<String>(x.DependsOn), StringComparer.OrdinalIgnoreCase);
+            var nodesToVisit = new Queue<GraphTraversalState<T>>(nodes.Where(x => x.DependsOn.Count == 0).Select(x => new GraphTraversalState<T>(x)));
+            while (nodesToVisit.Count > 0)
+            {
+                var currentNode = nodesToVisit.Dequeue();
+
+                // Invoke the callback on this node since it would execute next. The dependencies provided to the 
+                // callback is a fully recursive set of all dependencies for context on how a node would execute
+                // at runtime.
+                handleNode(currentNode.Node, currentNode.Dependencies);
+
+                // Now figure out which nodes would start as a result of this 
+                foreach (var nodeState in pendingNodes)
+                {
+                    for (Int32 i = nodeState.Value.Count - 1; i >= 0; i--)
+                    {
+                        if (nodeState.Value[i].Equals(currentNode.Node.Name, StringComparison.OrdinalIgnoreCase))
+                        {
+                            nodeState.Value.RemoveAt(i);
+
+                            // Make sure we include the completed nodes recursive dependency set into the dependent
+                            // node recursive dependency set for accurate hit detection.
+                            var traversalState = nodeLookup[nodeState.Key];
+                            traversalState.Dependencies.Add(currentNode.Node.Name);
+                            traversalState.Dependencies.UnionWith(currentNode.Dependencies);
+
+                            if (nodeState.Value.Count == 0)
+                            {
+                                nodesToVisit.Enqueue(traversalState);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        private class GraphTraversalState<T> where T : class, IGraphNode
+        {
+            public GraphTraversalState(T node)
+            {
+                this.Node = node;
+            }
+
+            public T Node { get; }
+            public ISet<String> Dependencies { get; } = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
+        }
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/IInputValidator.cs

@@ -0,0 +1,18 @@
+using System.ComponentModel;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    /// <summary>
+    /// Provides a contract validators must implement to participate in input validation.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public interface IInputValidator
+    {
+        /// <summary>
+        /// Validates the input value using the provided context.
+        /// </summary>
+        /// <param name="context">The current input validation context</param>
+        /// <returns>A result which indicates success or failure of the validation in addition to detailed reason on failure</returns>
+        InputValidationResult Validate(InputValidationContext context);
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/InputValidationContext.cs

@@ -0,0 +1,73 @@
+using System;
+using System.ComponentModel;
+using System.Runtime.Serialization;
+using GitHub.DistributedTask.Expressions;
+using GitHub.DistributedTask.Logging;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    /// <summary>
+    /// Provides the necessary context for performing input value validation.
+    /// </summary>
+    [DataContract]
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public class InputValidationContext
+    {
+        /// <summary>
+        /// Gets or sets an expression which should be used to validate <see cref="Value"/>.
+        /// </summary>
+        [DataMember(EmitDefaultValue = false)]
+        public String Expression
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether or not to evaluate the expression using <see cref="Value"/>.
+        /// </summary>
+        [DataMember(EmitDefaultValue = false)]
+        public Boolean Evaluate
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Gets or sets the options used during expression evalation.
+        /// </summary>
+        public EvaluationOptions EvaluationOptions
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Gets or sets the secret masker implementation.
+        /// </summary>
+        public ISecretMasker SecretMasker
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Gets or sets the trace writer implementation.
+        /// </summary>
+        public ITraceWriter TraceWriter
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Gets or sets the value which should be validated.
+        /// </summary>
+        [DataMember(EmitDefaultValue = false)]
+        public String Value
+        {
+            get;
+            set;
+        }
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/InputValidationResult.cs

@@ -0,0 +1,43 @@
+using System;
+using System.ComponentModel;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    /// <summary>
+    /// Provides information about the result of input validation.
+    /// </summary>
+    [DataContract]
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public class InputValidationResult
+    {
+        public InputValidationResult()
+        {
+        }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether or not the input value is valid.
+        /// </summary>
+        [DataMember(EmitDefaultValue = false)]
+        public Boolean IsValid
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Gets or sets a value indicating a detailed reason the input value is not valid.
+        /// </summary>
+        [DataMember(EmitDefaultValue = false)]
+        public String Reason
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// Provides a convenience property to return successful validation results.
+        /// </summary>
+        public static readonly InputValidationResult Succeeded = new InputValidationResult { IsValid = true };
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/InputValidator.cs

@@ -0,0 +1,44 @@
+using System;
+using System.ComponentModel;
+using GitHub.DistributedTask.Expressions;
+using GitHub.DistributedTask.Pipelines.Expressions;
+using GitHub.DistributedTask.WebApi;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    /// <summary>
+    /// Provides n validator implementation for task inputs.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public class InputValidator : IInputValidator
+    {
+        public InputValidationResult Validate(InputValidationContext context)
+        {
+            if (String.IsNullOrEmpty(context.Expression))
+            {
+                return InputValidationResult.Succeeded;
+            }
+
+            var result = new InputValidationResult();
+            try
+            {
+                var parser = new ExpressionParser();
+                var tree = parser.CreateTree(context.Expression, context.TraceWriter, namedValues: InputValidationConstants.NamedValues, functions: InputValidationConstants.Functions);
+                if (context.Evaluate)
+                {
+                    result.IsValid = tree.Evaluate<Boolean>(context.TraceWriter, context.SecretMasker, context, context.EvaluationOptions);
+                }
+                else
+                {
+                    result.IsValid = true;
+                }
+            }
+            catch (Exception ex) when (ex is ParseException || ex is RegularExpressionInvalidOptionsException || ex is NotSupportedException)
+            {
+                result.Reason = ex.Message;
+            }
+
+            return result;
+        }
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/NameValidation.cs

@@ -0,0 +1,59 @@
+using System;
+using System.ComponentModel;
+using System.Text;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public static class NameValidation
+    {
+        public static Boolean IsValid(
+            String name,
+            Boolean allowHyphens = false)
+        {
+            var result = true;
+            for (Int32 i = 0; i < name.Length; i++)
+            {
+                if ((name[i] >= 'a' && name[i] <= 'z') ||
+                    (name[i] >= 'A' && name[i] <= 'Z') ||
+                    (name[i] >= '0' && name[i] <= '9' && i > 0) ||
+                    (name[i] == '_') ||
+                    (allowHyphens && name[i] == '-' && i > 0))
+                {
+                    continue;
+                }
+                else
+                {
+                    result = false;
+                    break;
+                }
+            }
+
+            return result;
+        }
+
+        public static String Sanitize(
+            String name,
+            Boolean allowHyphens = false)
+        {
+            if (name == null)
+            {
+                return String.Empty;
+            }
+
+            var sb = new StringBuilder();
+            for (Int32 i = 0; i < name.Length; i++)
+            {
+                if ((name[i] >= 'a' && name[i] <= 'z') ||
+                    (name[i] >= 'A' && name[i] <= 'Z') ||
+                    (name[i] >= '0' && name[i] <= '9' && sb.Length > 0) ||
+                    (name[i] == '_') ||
+                    (allowHyphens && name[i] == '-' && sb.Length > 0))
+                {
+                    sb.Append(name[i]);
+                }
+            }
+            return sb.ToString();
+        }
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/ScriptTaskValidator.cs

@@ -0,0 +1,198 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Text.RegularExpressions;
+using GitHub.Services.Common;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    /// <summary>
+    /// Validates script tasks for bad tokens. For best performance, create one instance and reuse - this is
+    /// thread safe.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public sealed class ScriptTaskValidator
+    {
+        /// <param name="clientTokenPrv">
+        /// If supplied, combined with <see cref="BaseBadTokenProvider"/> to form a single set of bad tokens.
+        /// </param>
+        public ScriptTaskValidator(IBadTokenProvider clientTokenPrv = null)
+        {
+            var regexToMatch = new HashSet<Regex>(RegexPatternComparer.Instance);
+            var tokenToMatch = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+            var tokenPrvs = new List<IBadTokenProvider>(2) { BaseBadTokenProvider.Instance };
+            if (clientTokenPrv != null)
+            {
+                tokenPrvs.Add(clientTokenPrv);
+            }
+
+            foreach (IBadTokenProvider tokenPrv in tokenPrvs)
+            {
+                foreach (string pattern in tokenPrv.GetRegexPatternsToMatch())
+                {
+                    regexToMatch.Add(
+                        new Regex(pattern, RegexOptions.Compiled, matchTimeout: TimeSpan.FromMilliseconds(100)));
+                }
+
+                foreach (string staticToken in tokenPrv.GetStaticTokensToMatch())
+                {
+                    tokenToMatch.Add(staticToken);
+                }
+            }
+
+            m_regexesToMatch = regexToMatch.ToArray();
+            m_stringsToMatch = tokenToMatch.ToArray();
+        }
+
+        /// <summary>
+        /// Check a string for tokens containing "banned" patterns. This method is thread safe.
+        /// </summary>
+        public bool HasBadParamOrArgument(
+            string exeAndArgs,
+            out string matchedPattern,
+            out string matchedToken)
+        {
+            ArgumentUtility.CheckForNull(exeAndArgs, nameof(exeAndArgs));
+
+            string[] args = exeAndArgs.Split();
+
+            // Using for loops b/c they are measurably faster than foreach and this is n^2 
+            for (int i = 0; i < args.Length; i++)
+            {
+                string arg = args[i];
+
+                // Check static matches
+                for (int j = 0; j < m_stringsToMatch.Length; j++)
+                {
+                    string toTest = m_stringsToMatch[j];
+                    if (arg.IndexOf(toTest, StringComparison.OrdinalIgnoreCase) >= 0)
+                    {
+                        matchedPattern = toTest;
+                        matchedToken = arg;
+
+                        return true;
+                    }
+                }
+
+                // Check regexes
+                for (int j = 0; j < m_regexesToMatch.Length; j++)
+                {
+                    Regex toTest = m_regexesToMatch[j];
+                    if (toTest.IsMatch(arg))
+                    {
+                        matchedPattern = toTest.ToString();
+                        matchedToken = arg;
+
+                        return true;
+                    }
+                }
+            }
+
+            matchedPattern = null;
+            matchedToken = null;
+            return false;
+        }
+
+        // These are arrays for max perf when enumerating/indexing
+        private readonly Regex[] m_regexesToMatch;
+        private readonly string[] m_stringsToMatch;
+
+        public interface IBadTokenProvider
+        {
+            IEnumerable<string> GetRegexPatternsToMatch();
+
+            IEnumerable<string> GetStaticTokensToMatch();
+        }
+
+        /// <summary>
+        /// Static set of bad tokens we know about.
+        /// </summary>
+        private sealed class BaseBadTokenProvider : IBadTokenProvider
+        {
+            private BaseBadTokenProvider()
+            {
+            }
+
+            public IEnumerable<string> GetRegexPatternsToMatch()
+            {
+                // https://en.wikipedia.org/wiki/Base58
+                const string base58charPattern = "[1-9a-km-zA-HJ-NP-Z]";
+
+                // We expect arguments to begin with whitespace (i.e. --config-option bla) or an = (i.e.
+                // --config-option=bla). If whitespace, we'll split the string so there is none to start.
+                const string beginTokenDelimiter = "(^|=)";
+
+                // We always expect arguments to end with whitespace, so any match will be the end of the string
+                const string endTokenDelimiter = "$";
+
+                string wrapInDelimeters(string argument)
+                {
+                    return beginTokenDelimiter + argument + endTokenDelimiter;
+                }
+
+                // Avoid patterns than can cause catastrophic backtracking for perf reasons
+                // https://www.regular-expressions.info/catastrophic.html
+                return new[]
+                {
+                    // Monero wallets. See https://moneroaddress.org/
+                    // http://monero.wikia.com/wiki/Address_validation
+                    wrapInDelimeters("4" + base58charPattern + "{94}"),
+                    wrapInDelimeters("4" + base58charPattern  + "{105}"), 
+
+                    // Bitcoin wallets. See https://en.bitcoin.it/wiki/Address
+                    // Starts with 1 or 3, total 33-35 base58 chars
+                    wrapInDelimeters("[1-3]" + base58charPattern + "{32,34}"),
+                    // Starts with bc[1-16], then 39(?) to 87 (90-3) base32 chars
+                    // See: https://en.bitcoin.it/wiki/Bech32
+                    wrapInDelimeters("bc[0-9]{1,2}([0-9a-zA-Z]){39}"),
+                    wrapInDelimeters("bc[0-9]{1,2}([0-9a-zA-Z]){59}"),
+                };
+            }
+
+            public IEnumerable<string> GetStaticTokensToMatch()
+            {
+                return new[]
+                {
+                    // Begin known mining pools
+                    "xmr.suprnova.cc",
+                    "MoneroOcean.stream",
+                    "supportXMR.com",
+                    "xmr.nanopool.org",
+                    "monero.hashvault.pro",
+                    "MoriaXMR.com",
+                    "xmrpool.",
+                    "minergate.com",
+                    "viaxmr.com",
+                    "xmr.suprnova.cc",
+                    // End known mining pools
+        
+                    // Probable mining argument
+                    "--donate-level",
+        
+                    // Other probable mining processes
+                    "cpuminer",
+                    "cryptonight",
+                    "sgminer",
+                    "xmrig",
+                    "nheqminer"
+                };
+            }
+
+            public static readonly IBadTokenProvider Instance = new BaseBadTokenProvider();
+        }
+
+        private sealed class RegexPatternComparer : IEqualityComparer<Regex>
+        {
+            private RegexPatternComparer()
+            {
+            }
+
+            public bool Equals(Regex x, Regex y) => x.ToString() == y.ToString();
+            public int GetHashCode(Regex obj) => obj.GetHashCode();
+
+            public static readonly IEqualityComparer<Regex> Instance = new RegexPatternComparer();
+        }
+    }
+}

src/Sdk/DTPipelines/Pipelines/Validation/ValidationResult.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.Pipelines.Validation
+{
+    [DataContract]
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public sealed class ValidationResult
+    {
+        public PipelineEnvironment Environment
+        {
+            get;
+            internal set;
+        }
+
+        public IList<PipelineValidationError> Errors
+        {
+            get
+            {
+                if (m_errors == null)
+                {
+                    m_errors = new List<PipelineValidationError>();
+                }
+                return m_errors;
+            }
+        }
+
+        public PipelineResources ReferencedResources
+        {
+            get
+            {
+                if (m_referencedResources == null)
+                {
+                    m_referencedResources = new PipelineResources();
+                }
+                return m_referencedResources;
+            }
+        }
+
+        public PipelineResources UnauthorizedResources
+        {
+            get
+            {
+                if (m_unauthorizedResources == null)
+                {
+                    m_unauthorizedResources = new PipelineResources();
+                }
+                return m_unauthorizedResources;
+            }
+        }
+
+        internal void AddQueueReference(
+            Int32 id, 
+            String name)
+        {
+            if (id != 0)
+            {
+                this.ReferencedResources.Queues.Add(new AgentQueueReference { Id = id });
+            }
+            else if (!String.IsNullOrEmpty(name))
+            {
+                this.ReferencedResources.Queues.Add(new AgentQueueReference { Name = name });
+            }
+        }
+
+        internal void AddPoolReference(
+            Int32 id,
+            String name)
+        {
+            if (id != 0)
+            {
+                this.ReferencedResources.Pools.Add(new AgentPoolReference { Id = id });
+            }
+            else if (!String.IsNullOrEmpty(name))
+            {
+                this.ReferencedResources.Pools.Add(new AgentPoolReference { Name = name });
+            }
+        }
+
+        [OnSerializing]
+        private void OnSerializing(StreamingContext context)
+        {
+            if (m_errors?.Count == 0)
+            {
+                m_errors = null;
+            }
+
+            if (m_referencedResources?.Count == 0)
+            {
+                m_referencedResources = null;
+            }
+
+            if (m_unauthorizedResources?.Count == 0)
+            {
+                m_unauthorizedResources = null;
+            }
+        }
+
+        [DataMember(Name = "Errors", EmitDefaultValue = false)]
+        private List<PipelineValidationError> m_errors;
+
+        [DataMember(Name = "ReferencedResources", EmitDefaultValue = false)]
+        private PipelineResources m_referencedResources;
+
+        [DataMember(Name = "UnauthorizedResources", EmitDefaultValue = false)]
+        private PipelineResources m_unauthorizedResources;
+    }
+}